update copilot and refresh models (#584)

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>

Author: haoranpb

.github/workflows/copilot-evaluation.yml

@@ -18,14 +18,11 @@ on:
           - "claude-opus-4.5"
           - "claude-opus-4.6"
           - "claude-opus-4.6-fast"
+          - "gpt-5.4"
           - "gpt-5.3-codex"
           - "gpt-5.2-codex"
           - "gpt-5.2"
-          - "gpt-5.1-codex-mini"
-          - "gpt-5.1-codex"
-          - "gpt-5.1-codex-max"
           - "gpt-4.1"
-          - "gemini-3-pro-preview"
       category:
         description: "Evaluation category to run"
         required: false
@@ -115,7 +112,7 @@ jobs:
         run: dotnet tool install Microsoft.Dynamics.BusinessCentral.Development.Tools --version 17.0.30.49729-beta --global
 
       - name: Install GitHub Copilot CLI
-        run: npm install -g @github/copilot@0.0.411
+        run: npm install -g @github/copilot@1.0.2
 
       - name: Select PAT based on job index
         id: select-pat

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "bcbench"
-version = "0.3.0"
+version = "0.3.1"
 description = "Benchmarking tool for Business Central (AL) ecosystem, inspired by SWE-Bench"
 readme = "README.md"
 requires-python = ">=3.13"

src/bcbench/agent/copilot/agent.py

@@ -58,7 +58,6 @@ def run_copilot_agent(entry: DatasetEntry, model: str, category: EvaluationCateg
             "--disable-builtin-mcps",
             f"--model={model}",
             "--log-level=debug",
-            "--disable-parallel-tools-execution",
             f"--log-dir={output_dir.resolve()}",
             f"-p={prompt.replace('\r', '').replace('\n', ' ')}",
         ]

src/bcbench/cli_options.py

@@ -35,14 +35,11 @@
         "claude-opus-4.5",
         "claude-opus-4.6",
         "claude-opus-4.6-fast",
+        "gpt-5.4",
         "gpt-5.3-codex",
         "gpt-5.2-codex",
         "gpt-5.2",
-        "gpt-5.1-codex-mini",
-        "gpt-5.1-codex",
-        "gpt-5.1-codex-max",
         "gpt-4.1",
-        "gemini-3-pro-preview",
     ],
     typer.Option(help="Copilot model to use"),
 ]