feat: test edit evaluation

.env.example

@@ -0,0 +1 @@
+OPENAI_API_KEY=

evaluation/run_edit_task_evaluation.py → evaluation/run_edit_task.py

@@ -1,4 +1,6 @@
 from dotenv import load_dotenv
+from loguru import logger
+from tqdm import tqdm
 
 from design_bench.evaluator.compile import collect_compile_information
 from design_bench.evaluator.config import Task
@@ -8,6 +10,9 @@
 
 
 def main():
+    logger.remove()
+    logger.add(lambda msg: tqdm.write(msg, end=""))
+
     models = [
         # "claude-3-7-sonnet-20250219",
         "gpt-4o-2024-11-20",
@@ -42,7 +47,7 @@ def main():
             continue
         for mode in modes:
             collect_compile_information(
-                task_name=Task.EDIT,
+                task_name=Task.edit,
                 frame_work=frame_work,
                 implemented_framework_or_mode=mode,
             )

mise.toml

@@ -1,3 +1,4 @@
 [tools]
 node = "20.19"
 npm = "latest"
+"npm:@angular/cli" = "latest"

package.json

@@ -0,0 +1,8 @@
+{
+  "dependencies": {
+    "@babel/parser": "^7.28.6",
+    "@vue/compiler-dom": "^3.5.27",
+    "parse5": "^8.0.0",
+    "single-file-cli": "^2.0.83"
+  }
+}

pyproject.toml

@@ -8,8 +8,19 @@ authors = [
 ]
 requires-python = ">=3.12"
 dependencies = [
+    "clip",
+    "datasets>=4.5.0",
+    "loguru>=0.7.3",
     "openai>=2.16.0",
+    "opencv-python>=4.13.0.90",
     "python-dotenv>=1.2.1",
+    "retry>=0.9.2",
+    "scikit-image>=0.26.0",
+    "selenium>=4.40.0",
+    "torch>=2.10.0",
+    "torchvision>=0.25.0",
+    "tqdm>=4.67.1",
+    "webdriver-manager>=4.0.2",
 ]
 
 [build-system]
@@ -18,3 +29,6 @@ build-backend = "uv_build"
 
 [dependency-groups]
 dev = ["pytest>=6.0.0", "ruff>=0.1.5", "ty>=0.0.10"]
+
+[tool.uv.sources]
+clip = { git = "https://github.com/openai/CLIP" }

src/design_bench/evaluator/compile.py

@@ -1,10 +1,12 @@
-import re
-import os
 import json
+import os
+import re
+
+import numpy as np
 import tqdm
 from PIL import Image
-import numpy as np
-from config import *
+
+from .config import DesignBench_Path, Task
 
 
 def is_pure_white_image(image_path):
@@ -115,30 +117,32 @@ def collect_compile_information(task_name, frame_work, implemented_framework_or_
     json_path = ""
     base_folder = ""
 
-    if task_name == Task.GENERATION:
+    if task_name == Task.generation:
         json_path = (
-            f"./res/DesignGeneration/{frame_work}_{implemented_framework_or_mode}.json"
+            DesignBench_Path
+            + f"res/DesignGeneration/{frame_work}_{implemented_framework_or_mode}.json"
         )
         base_folder = (
             DesignBench_Path
-            + f"data/DesignGeneration/GenerationResults/{frame_work}-{implemented_framework_or_mode}"
+            + f"data/generation/GenerationResults/{frame_work}-{implemented_framework_or_mode}"
         )
 
-    if task_name == Task.EDIT:
+    if task_name == Task.edit:
         json_path = (
-            f"./res/DesignEdit/{frame_work}_{implemented_framework_or_mode}.json"
+            DesignBench_Path
+            + f"res/DesignEdit/{frame_work}_{implemented_framework_or_mode}.json"
         )
         base_folder = (
-            DesignBench_Path + f"data/DesignEdit/EditResults/{frame_work}-{frame_work}"
+            DesignBench_Path + f"data/edit/EditResults/{frame_work}-{frame_work}"
         )
 
-    if task_name == Task.REPAIR:
+    if task_name == Task.repair:
         json_path = (
-            f"./res/DesignRepair/{frame_work}_{implemented_framework_or_mode}.json"
+            DesignBench_Path
+            + f"res/DesignRepair/{frame_work}_{implemented_framework_or_mode}.json"
         )
         base_folder = (
-            DesignBench_Path
-            + f"data/DesignRepair/RepairResults/{frame_work}-{frame_work}"
+            DesignBench_Path + f"data/repair/RepairResults/{frame_work}-{frame_work}"
         )
 
     # if not os.path.isfile(json_path):