[ { "Paper_ID": "S01", "CiteKey": "kavita2026autonomous", "Title": "Autonomous CI/CD Quality Assurance Using LangGraph Multi-Agent Orchestration and Risk-Proportionate Human-in-the-Loop Control", "Authors_Year": "Kavita Jadhav, 2026", "Venue_Tier": "International Journal of Engineering and Computer Science", "DOI_URL": "10.18535/ijecs/v15i06.5563", "Core_Problem": "Asymmetric scaling between software development velocity and quality assurance capacity in CI/CD pipelines, causing degraded QA coverage during high-velocity periods and introducing regression risks.", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "LangGraph multi-agent orchestration comprising 14 specialist agents across four pipeline phases (Planning, Execution, Analysis, Reporting), decoupled from tools via 7 Model Context Protocol (MCP) servers, with continuous self-evaluation using DeepEval and RAGAS.", "Coordination_Control": "Hierarchical stateful pipeline orchestration via LangGraph state graph; risk-proportionate Human-in-the-Loop (HITL) gate suspending execution for human review when risk score >= 0.85; multi-LLM consensus scoring.", "Foundation_Model": "GPT-4o, Claude Sonnet 4.6, Gemini 1.5 Pro", "Benchmark_Dataset": "120 pull requests from 3 open-source Python repositories (REST API service: 42 PRs, data processing pipeline: 38 PRs, web app: 40 PRs; Jan-Jun 2026)", "Artifact_Type": "K11tech Agentic AI QA System", "Primary_Metrics_Exact": "Defect detection rate: 91.2%, False positive rate: 8.4%, F1 score: 0.913", "Secondary_Metrics": "Phase 2 parallel execution duration: 2.5 min vs serial: 18.4 min; Cohen's kappa: 0.84", "Failure_Modes": "Non-deterministic test failures passed forward unmodified; optimistic bias in HITL trigger precision due to single-dataset calibration; reviewer fatigue moderating review accuracy.", "Threats_Validity": "Internal validity: single reviewer construct for human review, threshold calibrated on evaluation repos; External validity: evaluated only on 3 Python repositories; Statistical conclusion validity: sample size limited to 120 PRs.", "Provenance_Tag": "FAKTA-DOK (papers/B1-001.txt Table 4: 91.2%/8.4%/F1 0.913, CI [0.872,0.945]; Code: github.com/K11-Software-Solutions/k11techlab-agentic-ai-qa-system)", "QA_Score": "9.5", "BID": "B1-001", "Decision": "INCLUDE-L2", "Role": "primary (QA orchestration, LangGraph)", "Extraction_Level": "fulltext", "txt_path": "papers/B1-001.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "peer-reviewed", "hitl", "langgraph", "rag", "orchestration" ] }, { "Paper_ID": "S02", "CiteKey": "chansa2026the", "Title": "The Wheel of Intelligence: Contract-Enforced Closed-Loop Architectures for Reliable LLM-Agent Systems", "Authors_Year": "Chansa Kabwe, University of South Africa", "Venue_Tier": "TechRxiv 2026 (preprint, CC-BY 4.0, 9 pages, not peer-reviewed)", "DOI_URL": "https://doi.org/10.36227/techrxiv.177205043.39321579/v1", "Core_Problem": "Open-loop LLM-agent execution makes failures hard to audit and unsafe actions hard to prevent; two failure modes: integration failures (weakly supported claims without provenance) and control failures (unsafe/irreversible tool actions without verification gate) [FAKTA-DOK fulltext p1]", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "WoI hub-and-spoke: central typed Knowledge Representation (KR) hub + 7 components (KR, Perception P, Memory M, Reasoning proposer pi_R, Self-Reflection verifier V_SR, Actuation A, Learning L fast/slow/consolidation); KR object tuple , TYPE in {Claim, Entity, Plan, ToolCall} [FAKTA-DOK p2-4]", "Coordination_Control": "Propose-verify-act closed loop (Algorithm 1): pi_R proposes, hierarchical V_SR gates (L0 syntax JSON/types, L1 deterministic safety policy, L2 semantic entailment judge), correction policy pi_fix retry/backoff/halt (QA max 3 retries; tools force safe report-error); 3 contracts C1 retrieval, C2 memory ingestion, C3 actuation [FAKTA-DOK p3-4]", "Foundation_Model": "Proposer gpt-5.2 default + high-reasoning-effort variant; verifier frozen gpt-5-mini; embeddings OpenAI text-embedding-3-small + FAISS; Learning component frozen [FAKTA-DOK p5]", "Benchmark_Dataset": "Domain 1: HotpotQA distractor multi-hop QA, 500 validation episodes deterministically seeded; Domain 2: sandboxed file-tool env (list/read/write/delete), 500 instructions (450 benign + 50 adversarial), filesystem actuation as irreversible action class [FAKTA-DOK p5]", "Artifact_Type": "Reproducible evaluation harness + anonymized config-driven YAML artifact bundle + per-episode JSONL logs with run metadata (Supplementary) [FAKTA-DOK p6-7]", "Primary_Metrics_Exact": "HotpotQA 500eps (Table 3): open gpt-5.2 F1 0.199 UCR 0.002 EFR 0.064; open High F1 0.210 UCR 0.022 EFR 0.040; WoI+gpt-5.2 F1 0.204 UCR 0.000 EFR 0.026; WoI+High F1 0.207 UCR 0.006 EFR 0.020. Tool sandbox (Table 4): open gpt-5.2 success 0.830 CAR 0.018; open High 0.828 CAR 0.032; WoI+gpt-5.2 0.734 CAR 0.000 (9 gate rejects); WoI+High 0.860 CAR 0.000 (19 rejects). Ablation (Table 5): No SR CAR 0.018 LC 1.0; +L0 0.018/1.0; +L1 0.000/1.0 [FAKTA-DOK p5-6]", "Secondary_Metrics": "High reasoning improves F1 (0.210) but raises open-loop unsupported claims (UCR 2.2%) mitigated by WoI to 0.6%; reasoning capability does not imply safety (High open CAR 3.2% ~2x default); LC>1 in QA from re-sampling, LC=1.0 in tools via forced safe-error; formal metrics LC/ARR/ISR/CAR + UCR/EFR defined Eq p4 [FAKTA-DOK p5-6]", "Failure_Modes": "Verifier false rejections/acceptances; incomplete policies or compromised verifiers weaken guarantees; extra verification cost/latency + QA re-sampling overhead [FAKTA-DOK Sec 10]", "Threats_Validity": "Learning frozen (control-only isolation); fixed dataset slice/retrieval/k/prompts/retry budget across conditions; safety conditional on chosen contracts; proprietary model APIs limit full reproducibility without same endpoints [FAKTA-DOK Sec 7/10/12]", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-023.txt Table 3/4/5 p5-6; preprint TechRxiv CC-BY, no public code URL)", "QA_Score": "8.5", "BID": "B1-023", "Decision": "INCLUDE-L2", "Role": "primary (closed-loop pattern)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-023.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint" ] }, { "Paper_ID": "S03", "CiteKey": "gabriel2026codeair", "Title": "CodeAir: A Multi-Agent Orchestration Framework for AI-Assisted Software Development", "Authors_Year": "Gabriel Avila Rangel, SevenApps LLC, Orlando FL", "Venue_Tier": "SSRN Electronic Journal 2026 (working paper, 6 pages, NOT peer-reviewed; manuscript Dec 2025; single-vendor evaluation)", "DOI_URL": "https://doi.org/10.2139/ssrn.5928934", "Core_Problem": "LLM coding assistants fail on complex multi-phase features: context-window loss of continuity, unstructured workflows, no systematic review; developers re-explain context and bridge workflow/CI gaps manually [FAKTA-DOK p1]", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "Tauri desktop (Rust backend + React frontend, ~10MB vs ~150MB Electron); 4 layers (UI / Tauri IPC Command / Agent / External: Claude Code CLI, git, Supabase, Stripe); 4 agents (CLAUDE.md Generator, ADR Creator, Developer, Code Review); ADR-based phased plans; context cycling (fresh Claude session per phase, re-inject CLAUDE.md/ADR/refs, persist via git commits+ADR); critical-op pattern gate (migration/DROP TABLE/DELETE FROM/rm -rf/force push/schema change) pausing for explicit human approval = HITL gate [FAKTA-DOK p2-4]", "Coordination_Control": "5 stages: setup -> feature planning with human ADR approval (may revise) -> sequential phase loop (prompt->execute->commit->advance, real-time monitoring) -> review loop (diff analysis, prioritized issues, fix phases until thresholds) -> PR delivery; strict phase ordering, ADR+git state [FAKTA-DOK p4]", "Foundation_Model": "Claude Code CLI (extended thinking mode) orchestrated via Rust agents [FAKTA-DOK p2-4]", "Benchmark_Dataset": "Production deployment across multiple projects over 3 months; internal before-after comparison (no external control group, no significance testing) [FAKTA-DOK Sec V]", "Artifact_Type": "Code https://github.com/seven-apps/codeair-agent [FAKTA-DOK p7]; data aggregated/anonymized upon request (not open)", "Primary_Metrics_Exact": "Feature time ~1 week -> ~4h (-96%); Table I: API+tests 3d->2.5h 91%, UI+state 4d->3h 94%, DB migration+API 5d->4h 92%, complex logic 7d->5h 97%; coverage 78% vs 62%, lint 2.3 vs 8.7 violations/feature, review cycles 1.2 vs 2.4; per-phase commits 100% vs ~40%, ADR docs 100% vs ~15%, review coverage 100% vs ~70% small feats [FAKTA-DOK Sec V]", "Secondary_Metrics": "Setup overhead 30-60 min; complex features need 2-3 ADR revisions; frequent pauses on multi-migration features; >10-phase features show diminishing returns [FAKTA-DOK Sec V-VI]", "Failure_Modes": "Setup cost; revision cycles; critical-op interruptions; scope sensitivity; vendor-evaluated (author is SevenApps, own product) [FAKTA-DOK Sec V-VI + funding/conflict p7]", "Threats_Validity": "Single-company production, no control, no statistical test, Claude Code/Tauri-specific generalizability untested, data not open; 96% claim is vendor before-after, not independent [FAKTA-DOK]", "Provenance_Tag": "FAKTA-DOK (fulltext PDF 6p 1.2MB LaTeX pdfTeX, txt 36.5KB, pdf_cache/B1-034.pdf+txt)", "QA_Score": "7.5", "BID": "B1-034", "Decision": "INCLUDE-L2", "Role": "supporting (industry practice)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-034.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint", "hitl" ] }, { "Paper_ID": "S04", "CiteKey": "sirui2024metagpt", "Title": "MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework", "Authors_Year": "Sirui Hong et al. (2023; 15 authors)", "Venue_Tier": "International Conference on Learning Representations (ICLR 2024)", "DOI_URL": "10.48550/arxiv.2308.00352", "Core_Problem": "Logic inconsistencies and cascading hallucinations caused by naively chaining LLM agents in complex multi-step software engineering tasks.", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "Meta-programming framework incorporating Standardized Operating Procedures (SOPs) into prompt sequences, assembly line paradigm with diverse agent roles (Product Manager, Architect, Project Manager, Engineer, QA Engineer), and publish-subscribe message pool.", "Coordination_Control": "SOP assembly-line workflow with role-specific structured communication interfaces, publish-subscribe message filtering, and executable code feedback loop.", "Foundation_Model": "GPT-4, GPT-3.5-turbo", "Benchmark_Dataset": "HumanEval (164 tasks), MBPP (427 tasks), and SoftwareDev (70 realistic software development tasks)", "Artifact_Type": "MetaGPT open-source multi-agent framework (https://github.com/geekan/MetaGPT)", "Primary_Metrics_Exact": "Pass@1: 85.9% on HumanEval, 87.7% on MBPP; SoftwareDev executability: 3.75/4", "Secondary_Metrics": "SoftwareDev running time: 541s, token usage: 31,255, code files: 5.1, human revision cost: 0.83; task completion rate: 100%", "Failure_Modes": "Cascading hallucinations in unconstrained dialogue; independent project execution lacking cross-project meta-learning; role constraints unable to dynamically adapt communication schemas.", "Threats_Validity": "Evaluation focused on small-to-medium greenfield projects rather than large legacy repositories; risk of benchmark data contamination in pre-trained LLMs.", "Provenance_Tag": "FAKTA-DOK (papers/B1-052.txt Sec 4.1-4.2: HumanEval 85.9%, MBPP 87.7%, exec 3.75/4; Code: github.com/geekan/MetaGPT)", "QA_Score": "9.5", "BID": "B1-052", "Decision": "INCLUDE-L2", "Role": "primary (foundational role-based framework)", "Extraction_Level": "fulltext", "txt_path": "papers/B1-052.txt", "year_derived": 2024, "tags_derived": [ "primary", "fulltext", "peer-reviewed" ] }, { "Paper_ID": "S05", "CiteKey": "yulin2026self", "Title": "A Self-Reflective Multi-Agent Collaboration Framework for Dynamic Software Engineering Tasks", "Authors_Year": "Yulin Huang, Georgia Institute of Technology", "Venue_Tier": "Preprints.org 2026 (preprint, CC BY 4.0, 11 pages, NOT peer-reviewed)", "DOI_URL": "https://doi.org/10.20944/preprints202603.0129.v1", "Core_Problem": "MetaGPT/AutoGen use static roles and fixed topologies, single-pass generation causes cascading hallucinations, and no experiential learning across tasks [FAKTA-DOK fulltext p1-2]", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "Eco-Evolve: Dynamic Orchestrator (complexity kappa(T)=sigmoid(We.Enc(T)+bc); topology G*=argmax P(G|T,kappa).Efficiency(G), Efficiency=exp(-0.1|V|-0.05|E|)) + System-1 working agents (PM/Architect/Developer) + System-2 Critic Agent at checkpoints + RAG Memory Bank + HER-Prompt self-evolution; 3 topologies chain (k<0.3) / star-mesh (0.3-0.7) / adaptive mesh (k>=0.7) [FAKTA-DOK p3-6]", "Coordination_Control": "Algorithm 1 (17 steps): complexity -> topology -> retrieve experiences -> augment prompts -> init agents -> per-phase execute -> Critic verify/revise -> collect trajectory -> HER memory update; Critic verify/synthesize/integrate phases; execution-based Score=0.4.Static+0.6.Dynamic [FAKTA-DOK p5-6]", "Foundation_Model": "GPT-4-Turbo for ALL frameworks (fair comparison); memory bank starts empty, fixed task ordering, no gradient updates (online continual agent, not fine-tuned) [FAKTA-DOK p6]", "Benchmark_Dataset": "SWE-bench Verified (500 GitHub issues, 12 Python repos) + DevBench (22 repos Python/C/C++/Java/JS, full lifecycle design/impl/test); vs MetaGPT, ChatDev, AutoGen; 3 runs, mean reported [FAKTA-DOK p6]", "Artifact_Type": "No public code/replication URL found in fulltext (grep github/GitHub empty) [FAKTA-DOK]; Algorithm 1 + equations + topology specs enable reimplementation", "Primary_Metrics_Exact": "SWE-bench Verified: Eco-Evolve 62.3% vs MetaGPT 49.2% (+13.1pp, +26.6% relative), ChatDev 45.8%, AutoGen 43.5%; DevBench: 73.5% vs MetaGPT 64.1% (+9.4pp, +14.7%), ChatDev 61.2%, AutoGen 58.7% (Table 1) [FAKTA-DOK p7]", "Secondary_Metrics": "Error Recovery 82.1% vs 58.4% MetaGPT; Iteration Efficiency 85.2% vs 68.7%; Ablation SWE-bench: full 62.3, w/o Critic 51.8 (-10.5), w/o Topology 54.2 (-8.1), w/o Memory 48.7 (-13.6), w/o HER 56.4 (-5.9); evolution iter0 56.4 -> iter10 62.3 (gap over best baseline 7.2 -> 13.1pp); topology use low-chain 78.2%, med-star 54.7%, high-mesh 75.9%; adaptive saves 23.7% tokens vs fixed mesh; cost 19.7k tokens/task (1.00x) vs MetaGPT 17.6k (0.89x) ~+12% overhead, Critic ~15% of tokens, HER retrieval <2%; case study 37/50 errors recovered (74%), avg 1.8 revisions [FAKTA-DOK p7-9]", "Failure_Modes": "Critic token overhead ~15%; cold-start with empty memory; task-ordering sensitivity; without reflection a List->Optional[List] regression propagates (caught only with Critic) [FAKTA-DOK p8-9]", "Threats_Validity": "Memory updated during eval (disclosed: no gradients, identical task order for all, baselines could cache but do not); single base LLM; 2 benchmarks only; no formal threats-validity section; no public artifact [FAKTA-DOK p9]", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-054.txt Sec 4.3 Table 1 p7; preprint, no public code URL)", "QA_Score": "8.5", "BID": "B1-054", "Decision": "INCLUDE-L2", "Role": "primary (adaptive SOTA, preprint-flagged)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-054.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "rag" ] }, { "Paper_ID": "S06", "CiteKey": "zeeshan2024codepori", "Title": "CodePori: Large-Scale System for Autonomous Software Development Using Multi-Agent Technology", "Authors_Year": "Zeeshan Rasheed, Abdul Malik Sami, Kai-Kristian Kemell, Muhammad Waseem, Mika Saari, Kari Systa, Pekka Abrahamsson (Tampere/Jyvaskyla)", "Venue_Tier": "arXiv cs.SE 2024 (preprint v2 2402.01411v2, submitted to Information and Software Technology; SSRN mirror 10.2139/ssrn.4979510)", "DOI_URL": "https://arxiv.org/abs/2402.01411v2", "Core_Problem": "LLM multi-agents handle dialogue but unproven on large/complex project code generation; single-pass and uncoordinated module splits fail past ~1000 LOC [FAKTA-DOK p1-2]", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "6 agents (Manager segmentation max 200 lines/module + auto-split; Dev01 initial; Dev02 optimize; Finalized01 QA; Finalized02 review; Verification final) via Algorithm 01; pair loops Dev1<->Dev2 x3 rounds and Fin1<->Fin2 rounds; instruction prompts with Response/Reflection/Code/Critique format; no-placeholder production-ready constraint [FAKTA-DOK p3-7]", "Coordination_Control": "Manager splits project -> per-module Dev1/Dev2 3-round refine -> Verification review -> Fin1/Fin2 refine -> save per-module file; accumulated-code alignment; >=3 cross-iterations per agent pair [FAKTA-DOK p3-7]", "Foundation_Model": "OpenAI API, model e.g. gpt-4 (Algorithm 01); temperature not specified [BELUM-TERDOK]", "Benchmark_Dataset": "HumanEval 164 (pass@k) vs MetaGPT/ChatDev/AlphaCode/Incoder/CodeGeeX/Codex/PaLM-Coder; manual 20 projects D1-D20 (selected by 6 developers, executed by first author) [FAKTA-DOK p8-13]", "Artifact_Type": "https://github.com/GPT-Laboratory/CodePori + replication package [26] (generated code, modifications, outputs, run instructions) [FAKTA-DOK p2/p13]", "Primary_Metrics_Exact": "HumanEval pass@1 89% (vs AlphaCode 18%, InCoder 17%, PaLM-Coder 37%, CodeX 48%, Fig 4); manual 17/20 = 85% pass (adjustments allowed; D12/D15/D19 fail on syntax/logic) [FAKTA-DOK p11-13]", "Secondary_Metrics": "Table 4 per-project: lines 94-1180, modules 1-9, bill $0.20-$2.56 (e.g. D7 debate system 1180 lines/40min/9 modules/$2.56; D13 flashcards 94/4/1/$0.20); large projects tens of minutes; small D9/D10/D13/D17/D20 ran unmodified; large-module projects always needed lib/path/model adjustments [FAKTA-DOK p13]", "Failure_Modes": "Complex logic/edge cases (3 fails); large-output memory omission mitigated by module splitting; HumanEval Python-only + image-worded gaps + leakage risk (acknowledged) [FAKTA-DOK Sec 4/6]", "Threats_Validity": "Explicit Threats to Validity section (dataset/external/internal); manual eval by first author counting adjusted code as pass; dated baselines (2022-23); Python-only [FAKTA-DOK Sec 6]", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-058.txt p11-13 Table 4 Fig 4; Code: github.com/GPT-Laboratory/CodePori; Zenodo 10.5281/zenodo.13755415)", "QA_Score": "8.0", "BID": "B1-058", "Decision": "INCLUDE-L2", "Role": "supporting (large-scale generation)", "Extraction_Level": "fulltext (arXiv surrogate for SSRN record)", "txt_path": "pdf_cache/B1-058.txt", "year_derived": 2024, "tags_derived": [ "supporting", "fulltext", "preprint" ] }, { "Paper_ID": "S07", "CiteKey": "marco2025sallma", "Title": "SALLMA: A Software Architecture for LLM-Based Multi-Agent Systems", "Authors_Year": "Marco Becattini, Roberto Verdecchia, Enrico Vicario (2025)", "Venue_Tier": "IEEE SATrends 2025 (closed; handle copy exists)", "DOI_URL": "https://doi.org/10.1109/satrends66715.2025.00006", "Core_Problem": "Single-LLM-agent architectures lack task customization, memory, ground-truth access; how to architect LLM-based multi-agent software systems is uncharted", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "SALLMA: two core layers - (i) Operational Layer for request intent management, real-time task execution, dynamic agent orchestration; (ii) Knowledge Layer storing metamodels/configurations for workflows and agents", "Coordination_Control": "Abstract-only: dynamic orchestration via Operational Layer + metamodel-driven Knowledge Layer", "Foundation_Model": "Mistral, LLaMA (PoC); others via Hugging Face/LangChain", "Benchmark_Dataset": "Abstract-only: proof-of-concept viability deployment, no controlled benchmark vs baseline [BELUM-TERDOK]", "Artifact_Type": "Abstract-only: PoC with Docker, Kubernetes, Python, LangChain, Hugging Face, Mistral, LLaMA, SQL/NoSQL; deployed for public administration info + business simulation", "Primary_Metrics_Exact": "No exact effectiveness numbers in abstract; viability via deployment claim only [FAKTA-DOK abstract]", "Secondary_Metrics": "None in abstract [BELUM-TERDOK]", "Failure_Modes": "Not reported in abstract [BELUM-TERDOK]", "Threats_Validity": "Single PoC deployment, no baseline, generalizability unknown; fulltext needed", "Provenance_Tag": "FAKTA-DOK (OpenAlex abstract, 238 words)", "QA_Score": "5.5", "BID": "B1-059", "Decision": "INCLUDE-L2-ABSTRACT-ONLY", "Role": "supporting (architecture pattern reference)", "Extraction_Level": "abstract-only", "txt_path": "", "year_derived": 2025, "tags_derived": [ "supporting", "abstract-only", "peer-reviewed", "orchestration" ] }, { "Paper_ID": "S08", "CiteKey": "wannita2025human", "Title": "Human-In-The-Loop Software Development Agents", "Authors_Year": "Wannita Takerngsaksiri et al. (2025; 10 authors)", "Venue_Tier": "IEEE/ACM International Conference on Software Engineering: Software Engineering in Practice (ICSE-SEIP 2025)", "DOI_URL": "10.1109/icse-seip66354.2025.00036", "Core_Problem": "Existing LLM multi-agent paradigms are evaluated solely on historical open-source benchmark datasets without human feedback across intermediate development stages, and lack real-world industrial deployment evaluation.", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "HULA (Human-in-the-Loop LLM-based Agents framework) deployed into Atlassian JIRA; comprises AI Planner Agent (file localization & step-by-step coding plan formulation) and AI Coding Agent (code & PR generation), supporting offline and live industrial workflows.", "Coordination_Control": "Human-in-the-Loop (HITL) dual-stage approval gates: plan review/approval gate before code generation and PR review/merge gate before production merging; feedback loop enables engineers to refine plans and code interactively.", "Foundation_Model": "GPT-4 (backbone for agent execution and LLM-as-a-Judge code similarity evaluation)", "Benchmark_Dataset": "Offline: SWE-bench (2,294 issues, 12 Python repos) and Atlassian internal dataset (369 JIRA issues, 94 repos, 10+ languages); Online: 663 production JIRA issues + practitioner survey (n=109)", "Artifact_Type": "HULA framework integrated into Atlassian JIRA (proprietary enterprise software tool)", "Primary_Metrics_Exact": "Perfect File Localization: 84% (SWE-bench) vs 15% (Internal dataset); Average Recall per issue: 86% (SWE-bench) vs 30% (Internal dataset); Code Similarity: 45% (SWE-bench) vs 30% (Internal dataset); Unit test pass rate: 31% (SWE-bench); Plan approval rate: 82% (433/527 approved); PR creation rate: 25% (95/376); PR merge rate: 59% (56/95 merged)", "Secondary_Metrics": "Workflow generation completion: 97% (SWE-bench) vs 100% (Internal dataset); Practitioner survey: 41% agreed plan aligned accurately, 61% agreed generated code was easy to read/modify; Human-judge correlation: r = 0.7 (n=203 issues)", "Failure_Modes": "Drastic drop in file localization accuracy on enterprise repos (15% vs 84%) due to short issue descriptions (median 75 tokens vs 295 tokens), repository diversity (94 repos), and multi-language codebases (10+ languages); code quality concerns remain for complex tasks.", "Threats_Validity": "Construct validity: code similarity measurement subjectivity (mitigated by r=0.7 correlation with human ratings); External validity: evaluation confined to Atlassian Jira ecosystem; Internal validity: dependence on GPT-4 backbone model.", "Provenance_Tag": "FAKTA-DOK (papers/B1-060.txt Sec IV Table III p6, Fig 4 p6-7; proprietary JIRA, no public URL)", "QA_Score": "8.5", "BID": "B1-060", "Decision": "INCLUDE-L2", "Role": "primary (HITL industrial deployment, JIRA)", "Extraction_Level": "fulltext", "txt_path": "papers/B1-060.txt", "year_derived": 2025, "tags_derived": [ "primary", "fulltext", "peer-reviewed", "hitl" ] }, { "Paper_ID": "S09", "CiteKey": "konrad2024towards", "Title": "Towards LLM-augmented multiagent systems for agile software engineering", "Authors_Year": "Konrad Cinkusz, Jarosław A. Chudziak (2024)", "Venue_Tier": "ACM 2024 (gold OA)", "DOI_URL": "https://doi.org/10.1145/3691620.3695336", "Core_Problem": "Agile SE needs efficient cognitive MAS support for analysis, planning, coordination, and team-structure optimization", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "CogniSim Ecosystem: MAS + LLM integration with Dynamic Context agent profiling + Theory of Mind collaboration; roles: developers, executors, quality checkers, methodology reviewers; outputs docs/models/UML diagrams", "Coordination_Control": "Abstract-only: analyze-propose-plan-validate pipeline; predefined quality/performance measures; team-configuration impact simulation", "Foundation_Model": "Not specified in abstract [BELUM-TERDOK]", "Benchmark_Dataset": "Abstract-only: case studies + simulations (domains/numbers not in abstract)", "Artifact_Type": "Abstract-only: ecosystem + team-configuration simulator; no URL in abstract", "Primary_Metrics_Exact": "No exact numbers in abstract; practical application claimed via case studies/simulations [FAKTA-DOK abstract]", "Secondary_Metrics": "None in abstract [BELUM-TERDOK]", "Failure_Modes": "Not reported in abstract [BELUM-TERDOK]", "Threats_Validity": "Cannot assess validity from abstract; fulltext needed", "Provenance_Tag": "FAKTA-DOK (OpenAlex abstract, 116 words; ACM gold OA per OpenAlex)", "QA_Score": "5.0", "BID": "B1-067", "Decision": "INCLUDE-L2-ABSTRACT-ONLY", "Role": "supporting", "Extraction_Level": "abstract-only", "txt_path": "", "year_derived": 2024, "tags_derived": [ "supporting", "abstract-only", "peer-reviewed" ] }, { "Paper_ID": "S10", "CiteKey": "dong2023agentcoder", "Title": "AgentCoder: Multi-Agent-based Code Generation with Iterative Testing and Optimisation", "Authors_Year": "Dong Huang et al. (2023; 6 authors)", "Venue_Tier": "arXiv (cs.CL / cs.SE)", "DOI_URL": "10.48550/arxiv.2312.13010", "Core_Problem": "Balancing code snippet generation with effective test case generation and execution is challenging; single-agent models and existing multi-agent systems suffer from hallucinated test cases, inaccurate feedback, and high token overhead.", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "AgentCoder: multi-agent code generation framework comprising Programmer Agent (code generation and refinement), Test Designer Agent (generates basic, edge, and large-scale test cases), and Test Executor Agent (executes code against tests and writes feedback).", "Coordination_Control": "Iterative test-driven feedback loop: Programmer and Test Designer operate independently from problem prompt; Test Executor runs generated code against tests in an isolated Python environment and routes execution traces/errors back to Programmer for iterative repair.", "Foundation_Model": "GPT-4, GPT-3.5-turbo, PaLM Coder, Claude-instant-1, CodeLlama (evaluated on 14 LLMs)", "Benchmark_Dataset": "HumanEval (164 tasks), MBPP (378 tasks), HumanEval-ET, and MBPP-ET", "Artifact_Type": "AgentCoder multi-agent framework (prompt templates and execution pipeline detailed in paper)", "Primary_Metrics_Exact": "Pass@1: 96.3% on HumanEval and 91.8% on MBPP (GPT-4), 79.9% on HumanEval and 89.9% on MBPP (GPT-3.5-turbo) (Table 1 p6); Token overhead: 56.9K (HumanEval) and 66.3K (MBPP) with GPT-4 [FAKTA-DOK pdf_cache/B1-068.txt]", "Secondary_Metrics": "Test generation accuracy (GPT-3.5-turbo, Table 4 p8): 87.8% (HumanEval), 89.9% (MBPP); Code line coverage (Table 5 p8): GPT-4 91.7% (HumanEval) / 92.3% (MBPP), GPT-3.5 89.5% (MBPP); Execution time: 228.7s (HumanEval), 365.9s (MBPP); Multi-agent vs single-agent pass@1 improvement: +10.4% to +16.5%", "Failure_Modes": "Inaccurate test generation by Test Designer can lead to incorrect code modification; syntax and runtime errors in initial code generation; diminishing returns beyond 2-3 refinement iterations.", "Threats_Validity": "Potential benchmark data contamination in pre-trained LLMs; evaluation confined to algorithmic function-level code benchmarks (HumanEval/MBPP) rather than multi-file repository projects.", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-068.txt Table 1 p6, Table 4-5 p8; preprint, prompts only)", "QA_Score": "8.0", "BID": "B1-068", "Decision": "INCLUDE-L2", "Role": "supporting (test-driven generation)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-068.txt", "year_derived": 2023, "tags_derived": [ "supporting", "fulltext", "preprint" ] }, { "Paper_ID": "S11", "CiteKey": "minh2025agilecoder", "Title": "AgileCoder: Dynamic Collaborative Agents for Software Development based on Agile Methodology", "Authors_Year": "Minh Huynh Nguyen et al. (2025; 4 authors)", "Venue_Tier": "IEEE/ACM International Conference on AI Foundation Models and Software Engineering (FORGE 2025)", "DOI_URL": "10.1109/forge66646.2025.00026", "Core_Problem": "Existing multi-agent software development frameworks oversimplify workflows into rigid waterfall steps and feed entire codebases into prompts, causing context window exhaustion, high cost, and failure on large-scale evolving projects.", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "AgileCoder: multi-agent software development framework based on Agile Methodology structured into Sprints, with specialized roles: Product Manager, Scrum Master, Developer, and Tester, powered by a Dynamic Code Graph Generator (DCGG) maintaining a Code Dependency Graph (CDG).", "Coordination_Control": "Sprint-based iterative backlog progression with targeted global message pool access; DCGG generates code dependency subgraphs for precise prompt context; three-step code review prompting strategy (Implementation Checks, Backlog Compliance, Bug Identification).", "Foundation_Model": "GPT-3.5-turbo, Claude 3 Haiku, GPT-4, CodeGeeX-13B, PaLM Coder-540B, DeepSeek-Coder-33B", "Benchmark_Dataset": "HumanEval (164 tasks), MBPP (427 tasks), and ProjectDev (10 real-world multi-step software development tasks with acceptance criteria)", "Artifact_Type": "AgileCoder open-source repository (https://github.com/FSoft-AI4Code/AgileCoder)", "Primary_Metrics_Exact": "HumanEval pass@1: 70.53% (GPT-3.5), 79.27% (Haiku), 90.85% (GPT-4); MBPP pass@1: 80.92% (GPT-3.5), 84.31% (Haiku); ProjectDev Executability: 57.79% (vs ChatDev 32.79%, MetaGPT 7.73%)", "Secondary_Metrics": "ProjectDev total running time: 444s (vs ChatDev 120s, MetaGPT 48s); Avg time/sprint: 306s (AgileCoder); Expenses: $0.44 (vs ChatDev $0.12, MetaGPT $0.02); Token usage: 36,818 (vs ChatDev 7,440, MetaGPT 3,029); Number of errors: 0 (vs ChatDev 6, MetaGPT 32); Average sprints: 1.64 [FAKTA-DOK pdf_cache/B1-070.txt Table 2]", "Failure_Modes": "Reliance on LLM code generation can produce suboptimal code for complex requirements; computational scaling cost of maintaining CDG graphs as codebase scales; exclusion of human agile team dynamics.", "Threats_Validity": "ProjectDev benchmark limited to 10 software tasks; potential data contamination in LLMs for HumanEval/MBPP; reliance on static AST parsing for code dependency graph extraction.", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-070.txt Table 1-2; Code: github.com/FSoft-AI4Code/AgileCoder)", "QA_Score": "9.5", "BID": "B1-070", "Decision": "INCLUDE-L2", "Role": "primary (agile-sprint orchestration, DCGG)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-070.txt", "year_derived": 2025, "tags_derived": [ "primary", "fulltext", "peer-reviewed", "orchestration" ] }, { "Paper_ID": "S12", "CiteKey": "dongming2024mare", "Title": "MARE: Multi-Agents Collaboration Framework for Requirements Engineering", "Authors_Year": "Dongming Jin et al. (2024; 4 authors)", "Venue_Tier": "arXiv (cs.SE)", "DOI_URL": "10.48550/arxiv.2405.03256", "Core_Problem": "Requirements engineering (RE) involves multifaceted collaborative phases (elicitation, modeling, verification, specification); existing deep learning approaches focus only on isolated subtasks rather than an integrated collaborative lifecycle.", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "MARE: multi-agent collaboration framework for RE comprising 5 specialized agents (Collector, Modeler, Checker, Documenter, Inspector) executing 9 actions across four phases: elicitation, modeling, verification, and specification, communicating via a shared workspace.", "Coordination_Control": "Shared workspace blackboard architecture where agents store and retrieve intermediate artifacts; Checker validates models against acceptance criteria and provides feedback for iterative refinement; Documenter structures SRS via templates.", "Foundation_Model": "GPT-3.5-turbo (ChatGPT)", "Benchmark_Dataset": "5 public evaluation cases for use case diagrams (ATM, Cafeteria, Library, Assembly, Time Monitor), 1 public dataset for goal models (PURE / GoalModelDataset), and 4 new evaluation cases for problem diagrams (Smart Home, Temperature, AC, Humidistat)", "Artifact_Type": "MARE replication package and support material (https://github.com/publicsubmission/MARE-support-material)", "Primary_Metrics_Exact": "Average F1 score on requirements modeling: 84.1% on problem diagrams (+15.4% over SOTA), 78.9% on use case diagrams (+23.8% over SOTA), 87.6% on goal models (+0.6% over SOTA)", "Secondary_Metrics": "Precision and Recall across 10 evaluation cases; Human evaluation by 3 RE inspectors across 3 quality dimensions (Table VII): Completeness 0.78-1.21 (avg 0.98), Correctness 1.53-1.85 (avg 1.92), Consistency 1.54-1.95 (avg 1.98); Multi-agent vs single-agent ablation (+1.1% average F1)", "Failure_Modes": "Reliance on predefined metamodels and prompt templates for structured modeling; difficulty resolving deep requirements inconsistencies not captured by rule-based acceptance criteria.", "Threats_Validity": "Subjectivity in human inspection evaluation (mitigated by 3-inspector averaging); limited to 10 evaluation cases; evaluated exclusively on GPT-3.5-turbo backbone.", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-073.txt Table 3-5, Table VII; preprint; Support: github.com/publicsubmission/MARE-support-material)", "QA_Score": "8.5", "BID": "B1-073", "Decision": "INCLUDE-L2", "Role": "supporting (requirements engineering MAS)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-073.txt", "year_derived": 2024, "tags_derived": [ "supporting", "fulltext", "preprint" ] }, { "Paper_ID": "S13", "CiteKey": "feiyang2026detecting", "Title": "Detecting and Repairing Role Drift in Multi-Agent Collaboration with Lightweight Protocols (Preprints.org preprint surrogate for IEEE B1-081)", "Authors_Year": "Feiyang Wang, Hengguang Cui, Linghao Yang, Chi Shing Lee, Zhongkang Li, Chenfeiyu Wen (UIUC/Brown/Chicago/Hunter/NYU)", "Venue_Tier": "Preprints.org 2026 (preprint, CC BY 4.0, 9 pages, NOT peer-reviewed; surrogate for IEEE CNML 2026 B1-081 whose IEEE fulltext was datacenter-blocked)", "DOI_URL": "https://doi.org/10.20944/preprints202603.0348.v1", "Core_Problem": "Role drift in extended collaboration: boundary violations (planner writes code), redundant work, conflicting decisions, futile debates; errors compound across rounds, forcing human intervention [FAKTA-DOK preprint p1-2]", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "RoleFix: (1) lightweight turn protocol [ROLE]/[COMMITMENT]/[NEED_FROM_OTHERS]; (2) hybrid detector (rule-based structural checks + LLM judge ALIGNED/MINOR/MAJOR_DRIFT, constrained generation); (3) self-repair reflect->reassign->resume with checkpoints (Algorithm 1, reassign after 3 consecutive drifting turns); drift formal D(ot,ri), cumulative Sdrift; taxonomy 4 types (BV/RW/CD/FD) from 500 logs, kappa 0.78, 12.3% multi-type [FAKTA-DOK p2-6]", "Coordination_Control": "Per-turn declaration gate + hybrid detect + 3-phase repair loop with checkpoint restore and subtask redistribution to appropriate role [FAKTA-DOK p5-6]", "Foundation_Model": "GPT-4, temperature 0.7; 4-agent teams (Planner/Coder/Reviewer/Tester); max 10 collaboration rounds [FAKTA-DOK p2/p7]", "Benchmark_Dataset": "SWE-bench Lite 100 tasks (bug fix + feature) + 50 custom research-workflow tasks (lit review/methodology/impl/docs); baselines No Protocol / Protocol Only / Protocol+Detection [FAKTA-DOK p7]", "Artifact_Type": "No public code/replication URL found in preprint [FAKTA-DOK]; Algorithm 1 + prompts + taxonomy codebook enable reimplementation", "Primary_Metrics_Exact": "RoleFix 76.1% success vs 52.3% No Protocol (+23.8pp); drift 6.1 vs 18.7 incidents/task (-67.4%); rework 11.8% vs 34.2%; time 19.2 vs 28.4 min (-32.4%) (Table 1) [FAKTA-DOK p4/p7]", "Secondary_Metrics": "Detection F1 hybrid 0.85 avg (BV 0.89, RW 0.86, CD 0.83, FD 0.81) vs rule 0.64 vs LLM-only 0.77 (Table 2); ablation vs No-Protocol baseline: protocol alone -33.7% drift, +detection further -28.2%, +repair additional -31.5% (non-additive); per-turn latency +8.3% (protocol ~50ms, rule 10ms, LLM judge ~200ms triggered only 23% turns); role-adherence trajectories Fig 2 [FAKTA-DOK p7-8]", "Failure_Modes": "Taxonomy validated on SE+research only, may miss other domains; LLM judge adds latency/cost on ambiguous turns [FAKTA-DOK p8]", "Threats_Validity": "Pilot logs from internal bug-fix datasets (50 tasks x 10 rounds, GPT-4); 100+50 eval tasks; GPT-4 only; no public artifact; IEEE version numbers may differ - use with preprint flag [FAKTA-DOK p2/p7-8]", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-081-preprint.txt Table 1-2 p4/p7-8; preprint surrogate for IEEE B1-081)", "QA_Score": "8.5", "BID": "B1-081P", "Decision": "INCLUDE-L2", "Role": "primary (drift-stability, preprint surrogate)", "Extraction_Level": "fulltext (preprint surrogate)", "txt_path": "pdf_cache/B1-081-preprint.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint" ] }, { "Paper_ID": "S14", "CiteKey": "hariharan2025agentic", "Title": "Agentic RAG for Software Testing with Hybrid Vector-Graph and Multi-Agent Orchestration", "Authors_Year": "Mohanakrishnan Hariharan, Seshu Babu Barma, Satish Arvapalli, Evangeline Y. Sheela (2025)", "Venue_Tier": "IEEE ICODSE 2025 (IEEE)", "DOI_URL": "https://doi.org/10.1109/icodse68111.2025.11351757", "Core_Problem": "Enterprise software testing bottlenecks: QEs spend 30-40% time creating testing artifacts; manual methods and basic RAG suffer from hallucinations, context fragmentation, and loss of business relationships.", "Scope": "Enterprise Quality Engineering (QE) test artifact automation (test plans, test cases, and QE metrics) and SAP migration testing.", "Proposed_Architecture": "Agentic RAG framework: Multi-Agent Orchestration Layer (Scope Analysis, Objective Extraction, Strategy Generation, Plan Synthesis, Integration Point Identification, Modernized Test Case, Compliance Validation Agents), SingleStore hybrid vector-graph knowledge base, and traceability framework.", "Coordination_Control": "Specialized autonomous multi-agent pipeline orchestration with hybrid vector-graph retrieval, cross-module relationship traversal, and bidirectional traceability tracking.", "Foundation_Model": "Gemini, Mistral, GPT-4", "Benchmark_Dataset": "Synthetic Test Dataset (5,000 curated scenarios: 1k simple, 1k complex logic, 1k integration, 2k regression) and Enterprise SAP Dataset (SAP S/4HANA migration: 1,000 existing test cases, 15 modules, 100+ T-codes).", "Artifact_Type": "Empirical Enterprise Automation System & Pipeline Architecture", "Primary_Metrics_Exact": "Test artifact accuracy: 94.8% (vs 65.2% Basic RAG, 76.5% Template, 81.7% GPT-4 direct, 92.3% manual); Completeness: 96.2%; Consistency: 95.7%; Traceability: 98.1%; Overall quality: 96.2%.", "Secondary_Metrics": "Artifact creation time reduced by 85% (240h to 36h per phase); projected 35% cost savings; 16-month accelerated go-live; 35% improvement in defect detection rate; 25,000 test cases with 98.7% functional coverage (vs 84% manual); 92% post-deployment production defect reduction.", "Failure_Modes": "Domain specialization bounds (employee/finance/SAP requiring retraining for novel domains); hybrid KB drift requiring continuous synchronization as business workflows evolve; enterprise system integration complexity.", "Threats_Validity": "Internal: proprietary evaluation datasets without public open-source replication package; Construct: LLM quality assessment dimensions partially based on internal QE rubrics; External: evaluated primarily within Apple Corporate Systems and SAP migration ecosystems.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "8.5", "BID": "B1-086", "Decision": "INCLUDE-L2", "Role": "primary (agentic RAG testing, enterprise)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-086.txt", "year_derived": 2025, "tags_derived": [ "primary", "fulltext", "peer-reviewed", "rag", "orchestration" ] }, { "Paper_ID": "S15", "CiteKey": "das2025multiagent", "Title": "A Multi-Agent RAG Framework for Regulatory Compliance Checking of Software Requirements", "Authors_Year": "Souvick Das, Novarun Deb, Nabendu Chaki, Agostino Cortesi (2025)", "Venue_Tier": "ACM Transactions on Software Engineering and Methodology (TOSEM)", "DOI_URL": "https://doi.org/10.1145/3785472", "Core_Problem": "Automated regulatory compliance checking of software requirements specifications (SRS) suffers from incomplete domain knowledge, legal nuances, and high risk of incorrect interpretations under purely automated LLM checking.", "Scope": "Regulatory compliance analysis of software requirement specifications against GDPR and EU Data Act policies.", "Proposed_Architecture": "Multi-Agent RAG framework comprising Compliance Checker (CC) Agent, Report Analyzer (RA) Agent, Vector DB, and layered Short-Term / Long-Term Memory Scheme with rule consolidation loops.", "Coordination_Control": "Collaborative multi-agent review between CC Agent and RA Agent, prioritized sequential memory retrieval (STM to LTM), and formal Human-in-the-Loop (HITL) intervention gate for expert verification and rule generalization.", "Foundation_Model": "Large Language Models via prompt engineering (Zero-shot, Few-shot, CoT prompting)", "Benchmark_Dataset": "28 use cases for GDPR and 20 use cases for EU Data Act across multiple software domains (e-commerce, FaceRecSpotify, enterprise systems, SaaS).", "Artifact_Type": "Empirical Software Compliance Framework + Memory Architecture Blueprint", "Primary_Metrics_Exact": "Requirements incompleteness detection correctness: Student evaluation: 69% correct, 25% partially correct, 6% incorrect (GDPR); 70% correct, 26% partially correct, 4% incorrect (Data Act). Expert evaluation: 66% correct, 21% partially correct, 13% incorrect (GDPR); 78% correct, 21% partially correct, 1% incorrect (Data Act across 20 cases) [FAKTA-DOK pdf_cache/B1-088_annexure.txt Sec 3 / Fig 4]", "Secondary_Metrics": "Qualitative HITL resolution analysis across 4 illustrative failure scenarios (legitimate interest, data portability vs trade secrets, anonymization, proportionality).", "Failure_Modes": "Over-reliance on 'public data' heuristic, semantic confusion between technical and legal definitions, failure to recognize power imbalances in consent, edge-case misclassification.", "Threats_Validity": "Evaluation partially reliant on crowdworkers/students alongside experts; subjective legal interpretation of regulatory ambiguity; limited to GDPR and Data Act domains.", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-088_annexure.txt Sec 3 / Fig 4; ACM TOSEM annexure)", "QA_Score": "8.0", "BID": "B1-088", "Decision": "INCLUDE-L2", "Role": "primary (compliance checking, HITL gate)", "Extraction_Level": "fulltext (annexure verified)", "txt_path": "pdf_cache/B1-088_annexure.txt", "year_derived": 2025, "tags_derived": [ "primary", "fulltext", "peer-reviewed", "hitl", "rag" ] }, { "Paper_ID": "S16", "CiteKey": "carbowitz2026spoq", "Title": "SPOQ: Specialist Orchestrated Queuing for Multi-Agent Software Engineering", "Authors_Year": "Royce Carbowitz, Dheeraj Kumar (2026)", "Venue_Tier": "arXiv (cs.SE)", "DOI_URL": "https://doi.org/10.48550/arxiv.2606.03115", "Core_Problem": "Multi-agent SE systems suffer from coordination overhead (sequential role-playing bottlenecks), quality control gaps, and limited human oversight.", "Scope": "Autonomous repository-level full-stack software development, task decomposition, and quality gate enforcement.", "Proposed_Architecture": "SPOQ: Specialist Orchestrated Queuing combining (1) wave-based topological dispatch on task DAGs, (2) dual validation gates (planning validation and code validation), and (3) Human-as-an-Agent (HaaA) integration, supported by a 3-tier agent hierarchy (Opus workers, Sonnet reviewers, Haiku investigators).", "Coordination_Control": "Wave-based topological DAG dispatch computed via dependency graphs; dual quality gates (pre-execution planning check + post-execution code check); Human-as-an-Agent (HaaA) interactive escalation.", "Foundation_Model": "Claude 3.5 Sonnet, Claude 3 Opus, Claude 3 Haiku (Anthropic), replicated on Qwen3.6-35B-A3B", "Benchmark_Dataset": "Controlled benchmark across 4 full-stack tasks; synthetic DAG scheduling benchmark; longitudinal deployment study across 17 repositories (8,589 commits, 1,822 completed tasks, 13,866 executed tests).", "Artifact_Type": "Open-source Multi-Agent SE Framework & Reference Implementation (GitLab repository)", "Primary_Metrics_Exact": "Wave dispatch speedup: 1.03-1.11 critical-path lower bound ratio (up to 14.3x on unbounded synthetic DAGs), 1.4x on 2-slot backend; Planning coverage: 93.0 to 99.75 (eliminating cyclic plans); Dual validation defect reduction: 0.34 to 0.20 defects/task with test pass rate lift from 91.25% to 99.75%; HaaA defect reduction: 0.47 to 0.03 defects/task (pass rate 96.5% to 99.75%); Deployment pass rate: 99.87% across 13,866 tests.", "Secondary_Metrics": "Parallelism potential lift: 31.0 to 75.25; Token cost efficiency across Opus/Sonnet/Haiku tiered routing.", "Failure_Modes": "Runaway retry loops (e.g. repeated dependency installation over 100x mitigated by max 3 retries), lock file contention (EBUSY errors), context window exhaustion, validation gaming, inter-agent coordination conflicts.", "Threats_Validity": "Internal: observational deployment study lacks matched controls (mitigated by controlled benchmarks); Construct: 20 validation metrics rely partly on LLM-based assessment; External: initial repository corpus shares internal development ecosystem.", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-093.txt p19 Tab5, p21 Tab6, p22 Tab7, p24 Tab8, p26 Tab10, p29 Sec6.7.1; GitLab: gitlab.com/kenth56/spoq)", "QA_Score": "9.0", "BID": "B1-093", "Decision": "INCLUDE-L2", "Role": "primary (queuing orchestration, SPOQ)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-093.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "orchestration" ] }, { "Paper_ID": "S17", "CiteKey": "tieu2026inference", "Title": "Inference-Time Graph Engineering for Multi-Agent LLM Workflows", "Authors_Year": "Katherine Tieu, Dongqi Fu, Yinglong Xia, Hong Li, Hong Yan, Jingrui He (2026)", "Venue_Tier": "arXiv (cs.AI / cs.SE)", "DOI_URL": "http://arxiv.org/abs/2609.05774v1", "Core_Problem": "Existing multi-agent LLM systems optimize static communication skeletons or rely on computationally expensive reinforcement learning/training-stage topology search, failing to adapt communication semantics and temporal progression dynamically at inference time.", "Scope": "Dynamic multi-agent workflow graph compilation and structured message passing for complex problem solving, coding, and tool-augmented reasoning.", "Proposed_Architecture": "ReActNet: Training-free inference-time compile-then-execute framework consisting of an LLM Controller (synthesizing instruction-typed temporal communication graphs across reasoning rounds), role-specialized agents, and an Aggregator/Synthesis node.", "Coordination_Control": "Compile-then-execute paradigm: decomposes reasoning into T temporal graph snapshots where directed edges specify natural-language instructions (what, why, and how to communicate); executed via structured message passing.", "Foundation_Model": "GPT-4o (controller and synthesis node), gpt-3.5-turbo (agent ablation)", "Benchmark_Dataset": "6 benchmark datasets across 3 categories: General Reasoning (MMLU), Math Reasoning (GSM8K, MultiArith, SVAMP, AQuA), Code Generation (HumanEval), plus GAIA assistant benchmark (validation split).", "Artifact_Type": "Multi-Agent Algorithmic Framework & Execution Engine", "Primary_Metrics_Exact": "HumanEval: 92.74% accuracy (+21.06% over vanilla 71.68%, outperforming MetaGPT 85.90% and G-Designer 89.90%); GSM8K: 94.45% (+9.05%); MultiArith: 100.00% (+6.85%); SVAMP: 94.67% (+7.49%); AQuA: 87.70% (+17.36%); MMLU: 86.93% (+4.79%); GAIA validation: 12.72% (outperforming G-Designer 10.91% and GPTSwarm 11.52%).", "Secondary_Metrics": "Inference wall-clock time and token consumption without training-time token overhead.", "Failure_Modes": "Inference cost scales with number of rounds T and agent count N; reliance on controller LLM competence to compile valid DAG instructions without runtime replanning; static edge instructions across a round.", "Threats_Validity": "Internal: Controller prompt sensitivity; Construct: evaluated on standard synthetic/algorithmic coding prompts (HumanEval) rather than full multi-file enterprise repository benchmarks (SWE-bench); External: proprietary OpenAI models (GPT-4o) tested.", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-094.txt p6 Tab1, p7 Tab4; no public code URL)", "QA_Score": "8.5", "BID": "B1-094", "Decision": "INCLUDE-L2", "Role": "primary (inference-time graph engineering)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-094.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint" ] }, { "Paper_ID": "S18", "CiteKey": "chen2026bilevel", "Title": "Bilevel Coordinated Reflection: A Game-Theoretic Approach to Multi-Agent LLM Systems", "Authors_Year": "Yihang Chen, Yuxiang Chen, Yuxuan Huang, Meng Fang, Weilin Luo, Jun Wang (2026)", "Venue_Tier": "arXiv (cs.AI)", "DOI_URL": "http://arxiv.org/abs/2609.02750v1", "Core_Problem": "Multi-agent LLM systems lack a unified theoretical account of coordination, memory improvement, and verification; unconditional/text-only reflection plateaus and cannot distinguish text-indistinguishable environments.", "Scope": "Game-theoretic orchestration and grounded reflection for multi-agent software engineering repair and coordination tasks.", "Proposed_Architecture": "Bilevel Coordinated Reflection framework: leader orchestrator (task decomposition) + follower workers (local-update potential game) integrated with Stochastic Reflective Memory Ascent (SRMA).", "Coordination_Control": "Bilevel coordination game where leader controls decomposition quality and equilibrium slack, while SRMA commits candidate memory reflections only when certified by an environment-grounded evaluation risk decrease (with confidence gating & re-anchoring).", "Foundation_Model": "Kimi K2.5, DeepSeek", "Benchmark_Dataset": "SWE-bench (500 instances), Overcooked (cramped_room, asymmetric_advantages, centre_pots), Resource Contest (RC).", "Artifact_Type": "Theoretical Framework & Algorithmic Tool (SRMA) with Open-Source Code (GitHub)", "Primary_Metrics_Exact": "SWE-bench (500 instances): Bilevel SRMA achieves 72.2% (361/500) on Kimi K2.5 vs 58.4% for Free-form MA and 70.8% public mini-SWE-agent reference; on DeepSeek achieves 71.4% vs 68.2% mini-SWE v2.", "Secondary_Metrics": "Switch time cut by 67.9% and post-shift regret cut by 67.0% under piecewise stationarity; verifier calls reduced by 63.6% (82 vs 225 calls) via confidence gating.", "Failure_Modes": "Free-form reflection plateaus or enters infinite drift under persistent-harm condition; text-only judges fail to detect environment distribution shifts (incurring 85.4 regret).", "Threats_Validity": "Guarantees assume bounded coupling and finite action sets; incomplete test suites guarantee monotonicity only for verifier risk rather than true task utility; substantial token overhead during multi-agent reflection.", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-095.txt p6 Tab3, p7 Tab4; Code: github.com/YihangChen9/Bilevel-Coordinated-Reflection)", "QA_Score": "9.0", "BID": "B1-095", "Decision": "INCLUDE-L2", "Role": "primary (bilevel reflection, SRMA)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-095.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "coordination" ] }, { "Paper_ID": "S19", "CiteKey": "rajib2026learning", "Title": "Learning What to Retain: Gated-Memory Routing for Efficient Collaboration in Multi-Agent LLM Systems", "Authors_Year": "Rakibul Hasan Rajib, Mengxin Zheng, Qian Lou (2026)", "Venue_Tier": "arXiv (cs.AI)", "DOI_URL": "http://arxiv.org/abs/2609.00237v1", "Core_Problem": "Routing agents from query alone fails to adapt to intermediate progress/errors, while routing from full execution history causes execution-history overload and inflates token cost.", "Scope": "Efficient dynamic memory management and role/backbone routing for multi-agent reasoning and code generation.", "Proposed_Architecture": "Gated-Memory Routing: Memory Write Gate (commits non-redundant steps), Retrieval Gate (supplies compact relevant subset to agents), Role Allocator, LLM Router, and Adaptive Halting Controller.", "Coordination_Control": "Conditioned step-level role and backbone routing over filtered execution memory; adaptive halting stops execution once sufficient evidence is accumulated.", "Foundation_Model": "Qwen-2.5 (14B, 32B), Llama-3.1 (8B), Llama-3.2 (3B), Mistral-Nemo (12B)", "Benchmark_Dataset": "HumanEval, MBPP, GSM-Hard, MATH, MMLU-Pro (5 reasoning and code generation benchmarks).", "Artifact_Type": "Multi-Agent Routing Framework & Open-Source Code (GitHub)", "Primary_Metrics_Exact": "Exceeds strongest baseline by 2.44 points average accuracy; HumanEval accuracy 89.84% (vs 85.16% query-only and 89.06% full-history) while cutting inference cost by 31.9% relative to strongest baseline (and over 50% vs full-history: 0.032 vs 0.068 cost); GSM-Hard accuracy 70.55% with 0.587 cost.", "Secondary_Metrics": "Inference token cost reduction of 43.86% vs MASRouter; leave-one-out ablation confirms LLM routing is most critical (14.0 drop on GSM-Hard without it).", "Failure_Modes": "Execution-history overload when write gate is disabled; performance drops on uniform short tasks (MBPP) where fixed chains suffice.", "Threats_Validity": "Evaluated on closed-domain verifiable benchmarks; compute proxy uses parameter count/token cost rather than hardware latency/memory pressure across all benchmarks.", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-097.txt p8 Tab1-2, p9 Tab3; Code: github.com/rajibrhasan/gated-memory-routing)", "QA_Score": "8.5", "BID": "B1-097", "Decision": "INCLUDE-L2", "Role": "primary (gated-memory routing)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-097.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint" ] }, { "Paper_ID": "S20", "CiteKey": "zhao2026towards", "Title": "Towards Fully Automated Medical Imaging Code Generation via Validation-based Context Engineering", "Authors_Year": "Zixiao Zhao, Jing Sun, Zhe Hou, Cheng-Hao Cai, Qian Liu, Mengze Li (2026)", "Venue_Tier": "ASE 2026 / arXiv (cs.CV / cs.SE)", "DOI_URL": "http://arxiv.org/abs/2608.29016v1", "Core_Problem": "General LLMs lack explicit domain knowledge and robust validation mechanisms for complex medical imaging code, requiring substantial human intervention to avoid catastrophic hallucination and syntax/semantic errors.", "Scope": "Automated domain-specific code generation and validation for medical image processing pipelines (2D/3D segmentation and classification).", "Proposed_Architecture": "AutoMedImg: Two-phase multi-agent framework comprising a Planning Phase (Dataset Analyser, Architecture Designer with semantic and formal verification) and Coding Phase (parallel module generation with static checking, execution testing, and assembly validation) with adaptive pipeline synthesis.", "Coordination_Control": "Hierarchical phased coordination with step-wise multi-stage validation gates (semantic validation, formal verification, static checks, execution testing) and cross-project knowledge reuse from shared memory.", "Foundation_Model": "Claude-4.5-Opus, GPT-5.2, Gemini-3-Pro, DeepSeek-V3.2, Qwen3-Coder", "Benchmark_Dataset": "6 medical imaging datasets across 4 categories: FLARE22, ACDC, BTCV, FLARE21, Promise12, OrganMNIST3D.", "Artifact_Type": "Multi-Agent System Framework & Pipeline Synthesis Architecture", "Primary_Metrics_Exact": "Zero human intervention (HI = 0 across all 6 datasets vs 1-9 for baselines); Dice score up to 0.9041 (ACDC) and 0.9517 (Promise12); Classification accuracy 99.38% (OrganMNIST3D) with Claude-4.5-Opus.", "Secondary_Metrics": "Pass@3 execution success; pipeline generation cost $20-$40 with average runtime 32 min vs manual 4-8 hours ($200-$600).", "Failure_Modes": "Error propagation across multi-step pipeline if intermediate validation fails; high token and generation overhead from multi-stage iterative checking.", "Threats_Validity": "Evaluation tailored to medical image processing domain standards; generation overhead is higher than single-turn direct generation.", "Provenance_Tag": "FAKTA-DOK (pdf_cache/B1-098.txt p9 Tab1, p10 Tab2, p11 Sec5.2; Code: github.com/SeanCho1996/AutoMedImg)", "QA_Score": "8.5", "BID": "B1-098", "Decision": "INCLUDE-L2", "Role": "primary (validation context engineering, medical)", "Extraction_Level": "fulltext", "txt_path": "pdf_cache/B1-098.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "coordination" ] }, { "Paper_ID": "S21", "CiteKey": "openjiuwen2026beyond", "Title": "openJiuwen: Beyond Static Harnesses for Long-Horizon Coding Agents", "Authors_Year": "Tao Yu, Xinyu Zhang, Qianqian Chen, Xiaoneng Xiang, Chia Kwangyang, Xingchen Huang, Ran Chen, Yangkai Ding, Zheng Wang, Yeo Boon Hong, Bingzheng Gan, Enrui Hu, Shuo Cheng, Deyang Li, Ruifeng Shi, Hongbo Wang, Qi Ye, Xuefeng Jin, Zhangchun Zhao (openJiuwen Team / Huawei Technologies Co., Ltd.) (2026)", "Venue_Tier": "arXiv (cs.AI / cs.SE)", "DOI_URL": "http://arxiv.org/abs/2608.27969v1", "Core_Problem": "Long-horizon coding agents face two core challenges: developer complexity in composing heterogeneous multi-agent topologies (Structural Composability) and inability to adapt runtime control based on evolving execution evidence (Runtime Adaptivity).", "Scope": "Agent harness design, composable execution substrates, and dynamic runtime control for long-horizon repository-level coding tasks.", "Proposed_Architecture": "openJiuwen: Layered execution harness featuring (1) Shared execution substrate, (2) Rail-based capability composition across single agents, delegated sub-agents, and Swarm Flow, and (3) Dynamic runtime adaptivity module adjusting context, feedback, and execution paths around a fixed model policy.", "Coordination_Control": "Hierarchical dual-loop orchestration (inner loop for atomic actions, outer loop for long-horizon coordination) with Rail-gated capability composition and dynamic evidence-driven task control.", "Foundation_Model": "Claude 4.5 Opus, GPT-5.6 Sol, Fable 5", "Benchmark_Dataset": "SWE-bench Verified (500 repository-level software issues) and Terminal-Bench 2.1 (heterogeneous long-horizon terminal tasks).", "Artifact_Type": "Open-Source Coding Agent Harness & Framework (GitHub)", "Primary_Metrics_Exact": "SWE-bench Verified: 82.60% resolved (Pass@1) with Claude 4.5 Opus (exceeding top leaderboard reference live-SWE-agent at 79.20% by +3.4 percentage points); Terminal-Bench 2.1: 87.19% accuracy with GPT-5.6 Sol (exceeding Claude Code at 83.8% by +3.39 percentage points).", "Secondary_Metrics": "Under model-matched Fable 5, Terminal-Bench 2.1 accuracy reaches 84.04% (vs Claude Code 83.8% and Terminus 2 80.4%); consistent performance gains across short, medium, and long task duration buckets.", "Failure_Modes": "State bloat and orchestration thrashing in unbounded long-horizon execution; tool misuse or context drift under complex multi-repo mutations.", "Threats_Validity": "Focuses on SWE-bench Verified and Terminal-Bench 2.1 benchmarks; variations in prompt engineering and tool interfaces across compared harness baselines.", "Provenance_Tag": "FAKTA-DOK (txt/B1-099.txt Table 1 l.1298-1312, Table 2 l.1376, Sec4.2-4.3; Code: github.com/openJiuwen-ai/jiuwenswarm)", "QA_Score": "8.0", "BID": "B1-099", "Decision": "INCLUDE-L2", "Role": "primary (long-horizon harness, composable rails)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-099.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "coordination", "orchestration" ] }, { "Paper_ID": "S22", "CiteKey": "seyedghorban2026observability", "Title": "Observability and Fault Injection for LLM-Based Multi-Agent Systems in Software Engineering", "Authors_Year": "Zahra Seyedghorban, Egor Klimov, Arie van Deursen, Annibale Panichella, Burcu Kulahcioglu Ozkan (2026)", "Venue_Tier": "IEEE / arXiv (cs.SE)", "DOI_URL": "http://arxiv.org/abs/2608.24271v1", "Core_Problem": "LLM-based multi-agent systems for SE are opaque black boxes that are difficult to inspect, debug, and systematically stress-test under controlled execution failures and communication breakdowns.", "Scope": "Distributed tracing, telemetry instrumentation, and controlled fault injection across inter-agent and LLM boundaries in SE multi-agent systems.", "Proposed_Architecture": "llmmas-otel: Lightweight, framework-agnostic tool combining OpenTelemetry-based hierarchical distributed tracing (session, segment, agent_step, a2a_send/receive, llm_call, tool_call) with a configurable execution-time fault injection layer.", "Coordination_Control": "Non-intrusive trace instrumentation via decorators and context managers; configuration-driven fault injection intercepting agent-to-agent communication (delay, drop, truncate), LLM calls (delay, rate-limit, timeout, network error, malformed output), and tool invocations.", "Foundation_Model": "Evaluated across agent workflows (e.g. ChatDev agents and demo pipeline using LLMs).", "Benchmark_Dataset": "Minimal multi-agent demo workflow and full-scale ChatDev software development multi-agent framework across 3 lifecycle phases (design, coding, testing).", "Artifact_Type": "Open-Source Observability & Fault-Injection Tool (GitHub / OpenTelemetry)", "Primary_Metrics_Exact": "Fault amplification factor (extra runtime / injected delay): in ChatDev, A2A message delay yields mean amplification of 59.2 (median 6.6) and LLM delay yields mean amplification of 48.1 (median 13.9); in minimal demo, A2A delay yields 1.295 mean amplification vs 1.053 for LLM delay.", "Secondary_Metrics": "Hierarchical trace span taxonomy (Table I); supported fault boundaries and fault catalog (Table II).", "Failure_Modes": "Delay cascade failure: minor perturbations at inter-agent communication boundaries magnify dramatically (up to ~60x) into severe workflow bottlenecks and downstream retries.", "Threats_Validity": "Evaluated on minimal demo and ChatDev framework; primarily measured delay-based fault amplification; full validation of non-deterministic drop/truncate perturbations left to future work.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "8.5", "BID": "B1-100", "Decision": "INCLUDE-L2", "Role": "primary (observability, fault injection)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-100.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "observability" ] }, { "Paper_ID": "S23", "CiteKey": "deoliveira2026developing", "Title": "Developing LLM-based Multi-Agent Systems in Software Engineering: A Mixed-Method Experience Report", "Authors_Year": "Mariama Celi Serafim De Oliveira, Motunrayo Osatohanmen Ibiyo, Marco Gianrusso, Claudio Di Sipio, Davide Di Ruscio, Phuong T. Nguyen (2026)", "Venue_Tier": "arXiv (cs.SE)", "DOI_URL": "http://arxiv.org/abs/2608.11965v1", "Core_Problem": "Developers face steep challenges in selecting frameworks, designing coordination rules, and defining agent roles for SE tasks; lack of empirical guidance on framework trade-offs.", "Scope": "Framework evaluation (AutoGen, AutoGPT, MetaGPT, CrewAI, ChatDev, Camel) on README.MD summarization / prompt optimization pipeline across GitHub repositories.", "Proposed_Architecture": "Comparative mixed-method evaluation pipeline: qualitative feature analysis across 6 C-dimensions + quantitative optimization pipeline comparing MAS frameworks with TS10/TS50 datasets.", "Coordination_Control": "Evaluates orchestration models across frameworks (centralized/hierarchical vs flat conversational); highlights that advanced control like agent telemetry and HITL gates remain severely limited in open-source frameworks.", "Foundation_Model": "gpt-4o-mini (2024-07-18), gpt-4o (2024-08-06)", "Benchmark_Dataset": "Curated GitHub repository dataset (README.MD and About pairs): filtered from 6,933 repos down to 925 (cosine similarity > 0.6), split into TS10 (10 samples), TS50 (50 samples), and ES (865 test samples).", "Artifact_Type": "Empirical Experience Report / Comparative Evaluation Framework & Replication Package", "Primary_Metrics_Exact": "ROUGE scores (ROUGE-1, ROUGE-2, ROUGE-L) across frameworks (e.g. variations between TS10 and TS50 showed no statistically significant difference in quality across frameworks).", "Secondary_Metrics": "Token usage (input/output tokens), Number of API requests, Total execution/usage time (s).", "Failure_Modes": "Execution history overload, prompt verbosity inflation in TS50 without generalization gain, lack of built-in telemetry/observability in current MAS frameworks.", "Threats_Validity": "Qualitative assessment based on author experience (subjective scoring); limited number of analyzed frameworks (6) and single SE domain task (README summarization); potential prompt bias.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "7.5", "BID": "B1-101", "Decision": "INCLUDE-L2", "Role": "supporting (mixed-method experience report, RQ4)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-101.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint", "hitl", "orchestration" ] }, { "Paper_ID": "S24", "CiteKey": "li2026morse", "Title": "MoRSE: Task-Oriented Multi-Agent System with Mixture of Role-Subtask Experts", "Authors_Year": "Peiwen Li, Shiyang Zhang, Yangtian Zhang, Sizhuang He, David van Dijk, Rex Ying (2026)", "Venue_Tier": "arXiv (cs.MA / cs.SE)", "DOI_URL": "http://arxiv.org/abs/2608.09251v1", "Core_Problem": "Existing MAS rely on coarse prompt-level differentiation without parameter adaptation, causing insufficient inter-agent heterogeneity, output redundancy, and performance bottlenecks in complex long-horizon tasks.", "Scope": "Task-oriented multi-agent code generation and scientific computing with dependency DAGs and parameter adaptation.", "Proposed_Architecture": "MoRSE: Task-Oriented Multi-Agent System combining (1) ToMAS (dependency-aware subtask DAG decomposition), (2) MoLE (Mixture of dual-factorized role-subtask LoRA Experts with prototype semantic router), and (3) HGRPO (Hierarchical GRPO with two-layer credit assignment).", "Coordination_Control": "Task-oriented DAG workflow with rule-based verifiers for step-level feedback; dynamic prototype-based semantic router isolates credit assignment between routing decisions and expert execution.", "Foundation_Model": "Qwen3-4B-Instruct, Llama-3.1-8B-Instruct, Gemma-4-31B-IT", "Benchmark_Dataset": "SRDD (Software Requirement-Driven Development benchmark) and SciCode (scientific coding benchmark with process-level and end-to-end problems).", "Artifact_Type": "Novel Multi-Agent Architecture + RL Training Framework (Code/Model Checkpoints)", "Primary_Metrics_Exact": "SRDD: Exec (%) 77.00% (All) / 72.50% (Test), ECI 0.718, Mean ECI Product 0.299 (Qwen3-4B); SciCode: Step Pass 25.44%, Mean Step Pass 27.22%, Problem Pass 6.25% (All) / 20.41% Step Pass (Test).", "Secondary_Metrics": "Inter-agent output redundancy reduction, trainable parameter ratio (0.048% - 0.170% across backbones), OOD generalization gains.", "Failure_Modes": "Credit assignment collapse under standard GRPO (destabilized surrogate); parameter saturation on ultra-large backbones (Gemma-4-31B showing smaller relative gain).", "Threats_Validity": "Subtask expert pool conservative; currently instantiates only two roles (execute, merge); SRDD training reward shares component sources with ECI metric; single random seed for main runs.", "Provenance_Tag": "FAKTA-DOK (txt/B1-102.txt Table 2-3; Code: github.com/lpwpower/MoRSE)", "QA_Score": "8.5", "BID": "B1-102", "Decision": "INCLUDE-L2", "Role": "primary (MoRSE, mixture of role-subtask experts)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-102.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint" ] }, { "Paper_ID": "S25", "CiteKey": "cutler2026curate", "Title": "CURATE: Leveraging LLM Agents to Compose, Catalog, and Deploy Reproducible Workflows", "Authors_Year": "Nolan Cutler, Chia-Chen Kuo, Nanda Velugoti, Kathryn Newhart, Renato Figueiredo (2026)", "Venue_Tier": "arXiv (cs.SE)", "DOI_URL": "http://arxiv.org/abs/2608.04270v1", "Core_Problem": "Coding agents focus narrowly on isolated code snippet generation and neglect the full workflow lifecycle (composition, cataloging, dependency resolution, execution, deployment, and reuse).", "Scope": "Lifecycle scientific/applied workflow automation, serverless pipeline deployment, and modular code reuse.", "Proposed_Architecture": "CURATE: Multi-agent system comprising Workflow Composition Agent (WCA), Module Catalog Agent (MCA), Module Generation Agent (MGA), and Workflow Deployment Agent (WDA) orchestrating 9 pipeline stages and 5 HITL gates.", "Coordination_Control": "LangGraph-based state machine with 5 Human-in-the-Loop (HITL) approval gates (Gate 1 Skeleton, Gate 2 Resolution, etc.) requiring explicit human sign-off/revision before downstream transitions.", "Foundation_Model": "Claude Opus 4.8 (Anthropic)", "Benchmark_Dataset": "SeBS-Flow serverless benchmark suite (4 workflows: Machine Learning, MapReduce, Video Analysis, Web Serving) and PyADM1 (Anaerobic Digestion Model No. 1 real-world scientific modeling).", "Artifact_Type": "Software Framework / Tool Prototype (LangGraph pipeline + FAIR catalog)", "Primary_Metrics_Exact": "100% end-to-end workflow deployment success across 6 test suites; token consumption per workflow vs single-agent Claude Code baseline (e.g. E5 vs E6 token halved via module reuse).", "Secondary_Metrics": "HITL intervention frequency, execution time, module reuse hit rate.", "Failure_Modes": "Single-agent baseline suffered from non-deterministic output and missing deployment harness; MAS token cost overhead when modules cannot be resolved from catalog.", "Threats_Validity": "Evaluation limited to 6 targeted benchmark/case workflows; reliance on proprietary Claude Opus 4.8; single-evaluator interaction for HITL gates.", "Provenance_Tag": "FAKTA-DOK (txt/B1-103.txt Table I; Code: github.com/nolcut/CURATE-prototype)", "QA_Score": "8.0", "BID": "B1-103", "Decision": "INCLUDE-L2", "Role": "primary (CURATE, HITL approval gates, reuse)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-103.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "hitl", "langgraph" ] }, { "Paper_ID": "S26", "CiteKey": "tukur2026agentic", "Title": "From AI Technical Debt to Agentic Technical Debt: A Systematic Mapping of Root Causes and Manifestations in Agentic AI Systems", "Authors_Year": "Muhammad Tukur, Hayatullahi Adeyemo, Tao Chen, Nour Ali, Anis Zarrad, Marco Agus, Rick Kazman, Rami Bahsoon (2026)", "Venue_Tier": "arXiv (cs.AI / cs.SE)", "DOI_URL": "http://arxiv.org/abs/2608.01001v1", "Core_Problem": "Traditional Technical Debt (TD) and AI Technical Debt (AITD) assume static component-level architectures, failing to capture emergent, cascading, and distributed liabilities in autonomous multi-agent systems.", "Scope": "Systematic mapping and formal conceptualization of Agentic Technical Debt (AgTD) across SE lifecycle and multi-agent coordination (Target for RQ4 + Snowballing).", "Proposed_Architecture": "AgTD Conceptual Framework & Mapping Model: Theory-informed 4-phase transformation methodology (Direct transformation, Contextual transformation, Manifestation expansion) mapping 31 AITDs across 7 root causes to agentic manifestations.", "Coordination_Control": "Taxonomy categorizes coordination failure liabilities: dynamic task orchestration fragility, communication feedback loops, consensus stalemates, unverified inter-agent boundaries, and memory degradation.", "Foundation_Model": "N/A (Systematic Mapping / Theoretical SE Framework)", "Benchmark_Dataset": "Literature corpus of 31 foundational AITD types across 7 root-cause categories mapped against agentic architectures and empirical agentic failure literature.", "Artifact_Type": "Systematic Mapping Study / Taxonomy & Architectural Guidelines for AgTD", "Primary_Metrics_Exact": "31 mapped AgTD types across 7 root-cause categories; 1:1 traceability preservation rate; transformation semantics distribution (direct d, contextual c, expansion e).", "Secondary_Metrics": "AI TRiSM alignment (Trust, Risk, Security, Governance dimensions), Sustainability Technical Debt impacts.", "Failure_Modes": "Memory drift/inconsistency, orchestration fragility, cascading cross-agent failures, uncontained prompt injection propagation, agentic hallucination compounding.", "Threats_Validity": "Conceptual mapping study without primary benchmark execution (secondary literature synthesis); validation rely on expert cross-checking rather than runtime testbed telemetry.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "7.5", "BID": "B1-104", "Decision": "INCLUDE-L2", "Role": "supporting (agentic technical debt mapping, RQ4)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-104.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint", "rag", "coordination", "orchestration" ] }, { "Paper_ID": "S27", "CiteKey": "bappy2026adversarial", "Title": "Adversarial Attacks in Multi-Agent LLM Pipelines: Unveiling Structural Vulnerabilities in Agentic AI Architectures", "Authors_Year": "Faisal Haque Bappy, Tahrim Hossain, Tarannum Shaila Zaman, Raiful Hasan, Kamrul Hasan, Tariqul Islam (2026)", "Venue_Tier": "arXiv (cs.CR / cs.SE)", "DOI_URL": "http://arxiv.org/abs/2608.00718v1", "Core_Problem": "Multi-agent LLM pipelines lack boundary verification primitives, creating implicit trust assumptions where malicious inputs propagate unchecked across agent boundaries and amplify failure.", "Scope": "Security vulnerability and adversarial robustness analysis in structured multi-agent pipelines (GAIA and SWE-Bench domains).", "Proposed_Architecture": "Trace-driven vulnerability discovery and boundary verification analysis framework evaluating 4 attack classes: A1 (Prompt Injection via Content), A2 (Consensus Poisoning), A3 (Plan Hijacking), and A4 (Sybil Agent Substitution).", "Coordination_Control": "Structural pipeline coordination (Centralized Planner, Deliberative Consensus, Sequential Hand-off); proves that vulnerability is an architectural property governed by inter-agent edge constraints rather than LLM size.", "Foundation_Model": "GPT-5-mini, Claude Sonnet 4.5, Kimi K2.5", "Benchmark_Dataset": "Annotated production execution traces derived from GAIA and SWE-Bench benchmarks (20 shared tasks x 3 escalating payload variants = 1,080 attack runs + 90 clean baseline runs).", "Artifact_Type": "Empirical Security Evaluation & Formal Boundary Verification Primitive Specification", "Primary_Metrics_Exact": "Attack Success Rate: A1 (0.61-0.72), A2 (0.79-0.83), A3 (0.81-0.86), A4 (0.76-0.78); Baseline clean accuracy: 0.80; Severe task accuracy degradation across all attacked pipelines.", "Secondary_Metrics": "Model robustness hierarchy (Claude Sonnet 4.5 > Kimi K2.5 > GPT-5-mini); Compliance-robustness trade-off metrics.", "Failure_Modes": "Unverified boundary crossing, planner hijacking from poisoned decomposition, cascading consensus failure, silent sybil agent impersonation.", "Threats_Validity": "Evaluated on 20 task traces across 3 frontier models; synthetic adversarial perturbation payloads; potential defense bypass techniques under adaptive adversaries.", "Provenance_Tag": "FAKTA-DOK (txt/B1-105.txt Sec IV-V; Code: github.com/SPaDeS-Lab/adversarial-llm-pipeline)", "QA_Score": "8.5", "BID": "B1-105", "Decision": "INCLUDE-L2", "Role": "primary (adversarial robustness, attack taxonomy)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-105.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "coordination" ] }, { "Paper_ID": "S28", "CiteKey": "fu2026agenticrepair", "Title": "AgenticRepair: Multi-Faceted Program Context Engineering for Agentic Vulnerability Repair", "Authors_Year": "Michael Fu, Qiyue Mei, Patanamon Thongtanunam, Kla Tantithamthavorn (2026)", "Venue_Tier": "arXiv cs.SE (pre-print)", "DOI_URL": "http://arxiv.org/abs/2607.29422v1", "Core_Problem": "Automated vulnerability repair (AVR) requires deep program context (cross-file data flows, memory safety invariants, crash semantics, and commit history) that general bug repair agents overlook, leading to incorrect or superficial patches.", "Scope": "Automated C/C++ vulnerability patch synthesis and verification across realistic security flaws.", "Proposed_Architecture": "Multi-faceted context-engineered multi-agent framework: 3 specialized context engineering subagents operating in parallel (Code-Structure Subagent, Runtime-Execution Subagent, Commit-History Subagent) coordinated into an episodic memory grounding a dedicated Repair Subagent (ToolCallingAgent via smolagents).", "Coordination_Control": "Hierarchical multi-agent orchestration with parallel context gathering; episodic memory sharing across subagents; max 20 steps for analysis subagents and max 75 steps for repair subagent; patch validation via compiler and AddressSanitizer/sanitizer-based test execution.", "Foundation_Model": "GPT-5.2 (gpt-5.2-2025-12-11; reasoning effort medium); evaluated also with GPT-5-mini and GPT-5-nano.", "Benchmark_Dataset": "SEC-Bench (300 real-world C/C++ vulnerability instances: 200 CVEs + 100 OSS vulnerabilities).", "Artifact_Type": "Autonomous multi-agent framework & replication package (open-source tool implementation, prompts, orchestration scripts).", "Primary_Metrics_Exact": "Success rate on SEC-Bench: 73.3% (220/300) overall, 75.0% (150/200 CVE), 70.0% (70/100 OSS) under strict sanitizer verification.", "Secondary_Metrics": "Ablation success rates (Strict): w/o Code-Structure 73.5% (-1.5%), w/o Runtime-Execution 74.5% (-0.5%), w/o Commit-History 73.0% (-2.0%), Single-agent scaffold 30.5% (-44.5%), GPT-5-mini 50.0% (-25.0%), GPT-5-nano 10.0% (-65.0%). Patch similarity: average normalized patch-line Jaccard 0.1177, content-only Jaccard 0.0561. 40% of successful repairs span multiple files.", "Failure_Modes": "Context explosion or incomplete multi-file tracing; single-agent degradation (dropping success rate from 75% to 30.5%); small model reasoning collapse (GPT-5-nano drops to 10.0%).", "Threats_Validity": "Internal: Data contamination mitigated via patch similarity showing low line/content Jaccard (0.1177 and 0.0561) vs gold patches; non-determinism of LLMs mitigated via open-source release; External: evaluation limited to C/C++ SEC-Bench instances, requiring further study for other languages and vulnerability categories.", "Provenance_Tag": "FAKTA-DOK (txt/B1-106.txt RQ1-RQ2; replication promised TSE, no public URL)", "QA_Score": "8.5", "BID": "B1-106", "Decision": "INCLUDE-L2", "Role": "primary (program context engineering, repair)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-106.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "orchestration" ] }, { "Paper_ID": "S29", "CiteKey": "shi2026bridging", "Title": "Bridging Behavior and Implementation: Automated Java Glue Code Generation for Behavior-Driven Development", "Authors_Year": "Xinyu Shi, Zhou Yang, An Ran Chen (2026)", "Venue_Tier": "arXiv cs.SE (pre-print)", "DOI_URL": "http://arxiv.org/abs/2607.19703v1", "Core_Problem": "In Behavior-Driven Development (BDD), writing and maintaining glue code connecting natural-language scenarios to project APIs is labor-intensive and error-prone due to underspecified behavior, evolving project code, and scattered BDD artifacts.", "Scope": "Automated glue code generation for Java BDD steps from open-source repositories.", "Proposed_Architecture": "AutoGlue: Hierarchical multi-agent framework structured via LangGraph into Behavior Interpreter, Developer agent, and two context retrieval subagents (BDD artifact retriever and project code retriever).", "Coordination_Control": "Hierarchical workflow separating behavior interpretation, semantic/embedding-based context retrieval (OpenAI text-embedding-3-small), and code synthesis; zero-shot / temperature-zero LLM-as-a-Judge validation.", "Foundation_Model": "GPT-5 mini (generation) + text-embedding-3-small (embeddings); LLM-as-a-judge evaluated with Gemini-2.5-Flash and Claude-Sonnet-4.5.", "Benchmark_Dataset": "1,307 BDD steps collected from 8 open-source Java projects (metasfresh, CxFlow, PSM, SYMON, JDI Light, SpringMVC Router, Worblehat, Datadog Java APM).", "Artifact_Type": "Multi-agent framework on LangGraph with prompt templates and evaluation suite.", "Primary_Metrics_Exact": "Functional alignment API F1: 0.660 (AutoGlue) vs 0.416 (Few-shot, +58.7% relative improvement) vs 0.284 (Plain Prompt). CodeBLEU: 0.641 (AutoGlue) vs 0.446 (Few-shot, +43.7% relative improvement) vs 0.412 (Plain Prompt).", "Secondary_Metrics": "Usability: 46.1% directly usable glue code (Exact match); partial match 35.0% (457/1,307, Table 4); API Precision: 0.660, API Recall: 0.714; METEOR: 0.655, ROUGE-L (RL): 0.614. [FAKTA-DOK txt/B1-107.txt Table 4]", "Failure_Modes": "Underspecified step intent; missing actions or parameter mismatches in complex step definitions; retrieval miss across deeply nested project hierarchies.", "Threats_Validity": "Internal: Data leakage from training data minimized by evaluating latest commit snapshots from wild open-source repos; prompt sensitivity in LLM-as-a-Judge mitigated via 5x repeated runs (<6% variance) and cross-model validation (Gemini vs Claude Sonnet); External: focused exclusively on Java BDD ecosystems (Cucumber-style), generalizability to other languages remains to be tested.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "8.5", "BID": "B1-107", "Decision": "INCLUDE-L2", "Role": "supporting (BDD glue generation)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-107.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint", "langgraph" ] }, { "Paper_ID": "S30", "CiteKey": "chen2026projagent", "Title": "ProjAgent: Procedural Similarity Retrieval for Repository-Level Code Generation", "Authors_Year": "Qihong Chen, Aaron Imani, Iftekhar Ahmed (2026)", "Venue_Tier": "arXiv cs.SE (pre-print)", "DOI_URL": "http://arxiv.org/abs/2607.08691v1", "Core_Problem": "Existing repository-level retrieval methods rely on lexical/semantic similarity, missing repository functions that share procedural logic and algorithmic steps but differ in naming, identifiers, or domain semantics.", "Scope": "Repository-level code generation across multi-file dependencies and project conventions in Python codebases.", "Proposed_Architecture": "ProjAgent: Procedural similarity retrieval pipeline decomposing target function into intermediate reasoning steps, projecting step hidden states onto a reasoning subspace to retrieve procedurally similar repository code, combined with semantic retrieval and a conservative static-analysis feedback loop.", "Coordination_Control": "Two-stage retrieval (procedural projection matching + dense/semantic retrieval) feeding an agentic code generation and repair loop guided by static analysis/compiler feedback; greedy decoding (temp=0, max 4096 tokens).", "Foundation_Model": "Qwen2.5-Coder-14B-Instruct (backbone for projection and generation); Claude Sonnet 4.6 (for procedural step pair dataset labeling).", "Benchmark_Dataset": "REPOCOD (980 problems across 11 real-world Python repositories including Astropy).", "Artifact_Type": "Repository-level code generation framework, subspace projection scripts, and evaluation harness.", "Primary_Metrics_Exact": "REPOCOD Pass@1: 41.14% (ProjAgent) vs 34.52% (SpecAgent, +6.62% absolute improvement), 28.83% (Dense), 26.58% (Sparse), and 14.98% (Same_File).", "Secondary_Metrics": "Procedural step retrieval precision/recall trade-offs: Config B (tau=0.65, k=2) achieves Promoted F1 0.090 and Leftover F1 0.940 across 9,598 step pairs.", "Failure_Modes": "Computational overhead of repository-wide decomposition; procedural false positives when algorithmic patterns differ in side effects; large repository search coverage bottlenecks.", "Threats_Validity": "Internal: Static-analysis feedback iteration limits; External: evaluated solely on Python repositories in REPOCOD; single backbone LLM (Qwen2.5-Coder-14B) due to high compute cost (980 full-repo executions per run); ablation on RQ3 conducted on Astropy repo.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "8.5", "BID": "B1-109", "Decision": "INCLUDE-L2", "Role": "primary (procedural retrieval, REPOCOD)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-109.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint" ] }, { "Paper_ID": "S31", "CiteKey": "du2026mlevolve", "Title": "MLEvolve: A Self-Evolving Framework for Automated Machine Learning Algorithm Discovery", "Authors_Year": "Shangheng Du, Xiangchao Yan, Jinxin Shi, Zongsheng Cao, Shiyang Feng, Zichen Liang, Boyuan Sun, Tianshuo Peng, Yifan Zhou, Xin Li, Jie Zhou, Liang He, Bo Zhang, Lei Bai (2026)", "Venue_Tier": "arXiv cs.AI (pre-print)", "DOI_URL": "http://arxiv.org/abs/2606.06473v1", "Core_Problem": "Machine Learning Engineering (MLE) agents suffer from inter-branch information isolation, memoryless search, and lack of hierarchical control, which prevents sustained self-evolution during long-horizon exploration.", "Scope": "Autonomous end-to-end machine learning engineering and scientific algorithm discovery on competitive benchmark tasks.", "Proposed_Architecture": "MLEvolve: Self-evolving multi-agent framework comprising Progressive Monte Carlo Graph Search (Progressive MCGS) with cross-branch reference edges and entropy schedule, Retrospective Memory (cold-start KB + dynamic global experience memory), and Hierarchical Planning with Adaptive Code Generation.", "Coordination_Control": "Hierarchical planning decoupling strategy from coding; graph-based exploration via dynamic reference edges between branches; progressive entropy-based exploitation schedule; feedback loop parsing runtime interpreter execution metrics.", "Foundation_Model": "Gemini-3.1-Pro-preview (primary backbone); evaluated also on GPT-5.5, DeepSeek-v4-Pro, Kimi-k2.5, Claude-Opus-4.6, and GPT-5.", "Benchmark_Dataset": "MLE-Bench (75 diverse Kaggle-level machine learning competition tasks) and AlphaEvolve mathematical algorithm optimization tasks (15 mathematical programming tasks).", "Artifact_Type": "Autonomous self-evolving multi-agent framework and codebase (https://github.com/InternScience/MLEvolve).", "Primary_Metrics_Exact": "MLE-Bench (12-hour budget, 75 tasks): Average Medal Rate 65.3±0.8% (All), Valid Submission Rate 100.0±0.0%, Gold Medal Rate 34.7±0.0%, Above-Median Rate 76.0±2.3%. Outperforms MARS+ (62.7% at 24h) and AIBuildAI (63.1% at 24h).", "Secondary_Metrics": "Complexity breakdown medal rates: Low 80.3±1.5%, Medium 64.0±0.9%, High 46.7±0.0%. On 15 mathematical programming tasks: superior beat ratio and convergence speed over AlphaEvolve.", "Failure_Modes": "Search space stagnation without Progressive MCGS (reverting to MCTS degrades performance significantly); context length consumption in dynamic global memory over long exploration horizons.", "Threats_Validity": "Internal: Sensitivity of progressive exploration schedule to entropy thresholding; External: benchmarks limited to MLE-Bench competitions and mathematical programming tasks; compute and token cost constraints in 12h-24h long-horizon search.", "Provenance_Tag": "FAKTA-DOK (txt/B1-116.txt Table 1 Sec4; Code: github.com/InternScience/MLEvolve)", "QA_Score": "9.0", "BID": "B1-116", "Decision": "INCLUDE-L2", "Role": "primary (self-evolving ML discovery)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-116.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "security" ] }, { "Paper_ID": "S32", "CiteKey": "amougou2026agentic", "Title": "An Agentic Approach Towards Replication Package Quality Evaluation", "Authors_Year": "Maximilian Alexander Amougou Mbida, Florian Angermeir (2026)", "Venue_Tier": "arXiv cs.SE (pre-print)", "DOI_URL": "http://arxiv.org/abs/2606.02006v2", "Core_Problem": "Artifact and replication package evaluation in empirical software engineering is manual, inconsistent, and bottlenecked, while open-science checklists are unstructured and unamenable to automated verification.", "Scope": "Automated evaluation of empirical software engineering research replication packages against operationalized open-science criteria.", "Proposed_Architecture": "Multi-agent replication package evaluator: Plan Agent, Retrieval Agent (git cloning, Zenodo download, doc parser), Orchestrator Agent, and Executor Agent generating evidence-grounded assessment reports.", "Coordination_Control": "Multi-agent coordination with role decomposition; retrieval pipeline fetching multi-source repositories; HITL planning gate with human reviewer; deterministic rubric-grounded scoring.", "Foundation_Model": "gpt-5-nano-2025-08-07 (retrieval and executor agents) + gpt-5-mini-2025-08-07 (plan and orchestrator agents).", "Benchmark_Dataset": "5 real-world replication packages evaluated across 31 operationalized criteria (consolidated from 380 requirements from 34 CORE A/A* venues and journals).", "Artifact_Type": "Multi-agent evaluation prototype, criteria taxonomy, and reproducibility evaluation dataset.", "Primary_Metrics_Exact": "Inter-run consistency: 91.4% overall (Data & Artifact Availability 94.8%, Ethical & Governance 94.0%); Correctness agreement with manual baseline: 75.4% micro-averaged agreement.", "Secondary_Metrics": "LLM-as-a-Judge semantic stability (1-4 scale): natural language reasoning 3.36/4.0, improvement suggestions 3.23/4.0, evidence path extraction 2.64/4.0. Pilot survey with 7 SE researchers confirmed perceived usefulness.", "Failure_Modes": "Misalignment with qualitative or mixed-method empirical packages (biases toward code-centric repositories); cognitive overhead during the HITL planning verification step; instability in extracting exact file paths for evidence.", "Threats_Validity": "Internal: Prompting sensitivity, LLM non-determinism, and potential implicit training data leakage; External: small sample size (5 replication packages, 7 researchers in survey), strong bias towards computational/code-centric artifacts rather than qualitative SE studies; Construct: automated criteria rubric approximation.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "7.0", "BID": "B1-117", "Decision": "INCLUDE-L2", "Role": "supporting (replication quality eval, preliminary)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-117.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint", "hitl", "coordination" ] }, { "Paper_ID": "S33", "CiteKey": "li2026bridging", "Title": "Bridging Requirements and Architecture: Multi-Agent Orchestration with External Knowledge and Hierarchical Memory", "Authors_Year": "Ruiyin Li et al. (2026; 9 authors)", "Venue_Tier": "arXiv cs.SE 2026", "DOI_URL": "http://arxiv.org/abs/2606.01385v1", "Core_Problem": "General MAS (e.g. MetaGPT) optimized for code generation lack architecture-specific workflows, domain knowledge grounding, and iterative evaluation for requirements-to-architecture design", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "MAAD: knowledge-driven MAS with 4 role-specific agents (Analyst, Modeler, Designer, Evaluator) + RAG injection of architectural standards/patterns + hierarchical 3-layer memory (short-term working, episodic long-term, semantic long-term) enabling iterative generate-evaluate-refine loop", "Coordination_Control": "Hierarchical staged pipeline Analyst->Modeler->Designer->Evaluator with cross-stage Evaluator gates enforcing traceability/consistency/QA satisfaction; closed-loop refinement via hierarchical memory reuse", "Foundation_Model": "GPT-5.2, Qwen3.5 (397B), DeepSeek-R1 (671B), Llama3.3 (70B)", "Benchmark_Dataset": "10 SRS cases (cyber-physical, PURE + industrial sources) vs MetaGPT baseline; 6-architect practitioner interviews; 4-LLM backbone comparison", "Artifact_Type": "Multi-agent architecture design framework + replication package (prompts, scripts, 10-case outputs)", "Primary_Metrics_Exact": "Architecture structural metrics across 10 cases vs MetaGPT (Qwen3.5 backbone, Table 3): Structural Complexity SC MAAD 0.01-0.05 vs MetaGPT 0.13-0.23; Interface Complexity IC MetaGPT 0.00 all cases (fails contract modeling) vs MAAD 1.31-2.40; RAG vs no-RAG ablation (Table 4): coupling density CCD decreases in 6/10 projects (e.g. C2C 2.91->2.29, Case 3.78->2.29, MCS 3.32->2.01); cohesion tends to decrease with fragmentation, behavioral state complexity rises [FAKTA-DOK txt/B1-119.txt Table 3-4 / Sec 5.1]", "Secondary_Metrics": "Backbone ranking: GPT-5.2 and Qwen3.5 superior on modularity/completeness vs DeepSeek-R1/Llama3.3; 6 practitioners (5y + 10y+ experience) rate artifacts well-structured, principle-aligned, useful as augmented assistant requiring human co-design", "Failure_Modes": "RAG over-fragmentation lowering class-level cohesion in some projects; weaker backbones (Llama3.3 SC=0.17 dense deps) expose reasoning deficits RAG cannot fully compensate; explainability gaps noted by practitioners", "Threats_Validity": "Internal: no causal inference design, LLM prompt sensitivity; External: 10 SRS domains + ISO 42010 grounding but domain-specific practices may not generalize; Construct: no single metric captures architectural quality, subjectivity mitigated by mixed metrics + expert review", "Provenance_Tag": "FAKTA-DOK (txt/B1-119.txt Sec5.1 Table 3-4; replication cited Li et al. 2026b, no direct URL)", "QA_Score": "8.0", "BID": "B1-119", "Decision": "INCLUDE-L2", "Role": "primary (requirements-architecture orchestration)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-119.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "rag", "orchestration" ] }, { "Paper_ID": "S34", "CiteKey": "yang2026when", "Title": "When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding", "Authors_Year": "Xu Yang et al. (2026; 6 authors)", "Venue_Tier": "arXiv cs.SE 2026", "DOI_URL": "http://arxiv.org/abs/2606.00953v1", "Core_Problem": "Naive parallelism in multi-agent coding degrades quality and inflates cost/latency (peer-query halts, speculative interfaces, cross-file contract violations) instead of improving throughput", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "Co-Coder: orchestrator building weighted file-dependency graph (vertex cost + symbol-sharing edge weights, Repository Interface Blueprint), isolating structural hub files, Infomap community detection partitioning, dependency-aware parallel scheduler with bounded test-feedback repair loop", "Coordination_Control": "DAG-based cohesion partitioning + dependency-respecting parallel execution avoiding layer-barrier sync; conflicting edits avoided by construction; re-dispatch only failing groups per repair round", "Foundation_Model": "Consistent base model across Sequential/File-parallel/Co-Coder (OpenHands SDK); Claude Code with Agent Teams as external baseline", "Benchmark_Dataset": "DevEval and CodeProjectEval repository-level coding benchmarks vs Sequential, File-based Parallel, Claude Code with Agent Teams (all on consistent base model)", "Artifact_Type": "Orchestration framework on OpenHands SDK + partitioning/scheduling implementation", "Primary_Metrics_Exact": "DevEval avg pass rate 68.1% vs 56.8% sequential (+11.3pp), latency 442s vs 800s (-45%), cost $0.18 vs $0.25 (-28%); CodeProjectEval 34.1% vs 20.1% sequential (+14.0pp), vs 23.3% file-parallel (+10.8pp), cost -35%, latency -45 to -52%; file-parallel cost +60% ($1.65 vs $1.03) for +3.2pp only", "Secondary_Metrics": "Pareto frontier across pass rate/latency/cost; per-repo partition stats; edge density explains larger CodeProjectEval gains; latency-correctness trade-off analysis", "Failure_Modes": "File-parallel inflates cost +44-60% for negligible gain; Claude Code Agent Teams lowest latency but lowest pass rate (DevEval 54.1%, CodeProjectEval 16.3% below sequential); gains depend on edge density, smaller on loosely-coupled projects", "Threats_Validity": "Internal: blueprint quality control dependence on static analysis; External: 2 benchmarks (DevEval formerly DevBench naming note), generalization beyond Python-leaning repos limited; Construct: wall-clock/API-cost hardware/prompt variance", "Provenance_Tag": "FAKTA-DOK (txt/B1-120.txt Table 1-2 Sec5; Code: github.com/Flitternie/CoCoder)", "QA_Score": "9.0", "BID": "B1-120", "Decision": "INCLUDE-L2", "Role": "primary (cohesion-aware partitioning, parallelism)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-120.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint" ] }, { "Paper_ID": "S35", "CiteKey": "ashrafi2026how", "Title": "How Generation Architecture Shapes Code Complexity in Multi-Agent LLM Systems: A Paired Study on HumanEval", "Authors_Year": "Nazmus Ashrafi, 2026", "Venue_Tier": "arXiv cs.SE 2026, Independent Researcher", "DOI_URL": "http://arxiv.org/abs/2606.00308v1", "Core_Problem": "MAS code generation evaluated almost exclusively on functional correctness (pass@1); whether orchestration architecture itself inflates structural complexity and which layer carries the cost is unexamined", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "Six multi-agent generation configurations (Basic, AC analyst-coder, ACT +tester, Debugger, AC+Debugger, ACT+Debugger) under paired within-task design; no new framework, controlled architectural ablation", "Coordination_Control": "Pipeline orchestration layers under test: analyst-coder split, tester addition, runtime debugger loop; paired-block design isolates layer cost contribution", "Foundation_Model": "GPT-4o family pair: flagship + cost-efficient mini (gpt-4o-mini-2024-07-18)", "Benchmark_Dataset": "Full HumanEval 164 tasks x 6 architectures x 2 GPT-4o-family models = 1968 paired observations, all-completions and passing-only conditions", "Artifact_Type": "Empirical pipeline + paired statistical analysis package (Friedman omnibus, Wilcoxon signed-rank Holm, Kendall W, rank-biserial effect sizes, RADON metrics)", "Primary_Metrics_Exact": "Two indistinguishable complexity clusters separated by 50-130% gap, same partition in both models and both conditions; pass@1 narrow band 84-92% (e.g. ACT 84.15% gpt-4o-mini); Debugger and AC+Debugger tied highest pass@1 in both models; all 10 Friedman omnibus tests reject null; complexity-pass@1 correlation negative per metric/model", "Secondary_Metrics": "Five RADON metrics (SLOC, cyclomatic CC, Halstead Volume/Difficulty/Effort); Friedman mean-rank diagrams; finding: report complexity alongside pass@1 when comparing generation architectures", "Failure_Modes": "Heavy cluster buys no pass@1 advantage; analyst-coder split inflates complexity, tester re-inflates, debugger alone does not inflate and actively deflates on analyst-coder background; leanest matches/beats heaviest on accuracy", "Threats_Validity": "Internal: listwise deletion for complete blocks, prior-work pass@1 drift up to several points; External: narrow model panel (2 closed-source GPT-4o family), HumanEval algorithmic prompts only; Construct: RADON proxies, not defect/review-cost ground truth", "Provenance_Tag": "FAKTA-DOK (txt/B1-121.txt Sec IV Table V-VI Fig2/7; no public repo)", "QA_Score": "8.0", "BID": "B1-121", "Decision": "INCLUDE-L2", "Role": "supporting (complexity-vs-architecture study)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-121.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint", "orchestration" ] }, { "Paper_ID": "S36", "CiteKey": "ahmed2026agentmodernize", "Title": "AgentModernize: Preserving Business Logic in Legacy Modernization with Multi-Agent LLMs and Behavioral Specification Graphs", "Authors_Year": "Sheikh Nazib Ahmed, Marnim Galib (2026)", "Venue_Tier": "arXiv cs.SE 2026, UT Arlington", "DOI_URL": "http://arxiv.org/abs/2605.17535v2", "Core_Problem": "Legacy modernization treated as syntax translation (COBOL->Java, PL/SQL->Python) loses implicit rules, edge cases, cross-module constraints that only surface as production failures", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "AgentModernize: 4-agent pipeline (Extractor, Specification, Modernization Transformer/code generation, Validator) with Behavioral Specification Graph (BSG) intermediate artifact forcing explicit inspectable business rules + feedback loop re-invoking transformer up to 3 iterations on BER divergence", "Coordination_Control": "Sequential extract->specify->generate->validate with BSG gate before code generation; BER-driven bounded feedback loop (max 3); selective iteration helps S4/S6/S7 but regresses S2/S8", "Foundation_Model": "GPT-4o-mini (primary), GPT-4o, Codex; cost mini $5, GPT-4o $12, Codex $18", "Benchmark_Dataset": "LegacyModernize-8 (8 synthetic telecom/banking scenarios) under fair protocol (each method tests from own API surface, 3 trials, temp 0.0) vs SP-LLM and CoT-LLM; cross-model study", "Artifact_Type": "Multi-agent modernization pipeline + LegacyModernize-8 benchmark + BSG extraction/evaluation harness", "Primary_Metrics_Exact": "Mean BER (gold-test pass rate): AgentModernize with feedback 23.0% (non-zero 5/8, up to 53.3%), without feedback 23.8%; SP-LLM 12.4% (2/8), CoT-LLM 4.5% (2/8); BSG rule recall 92.3% mean at 90.2% precision (S1/S6 100% recall, S-lowest 76.9% missing 3 SLA rules)", "Secondary_Metrics": "Per-scenario BER table (S4 25.0%, S5 27.8%, S8 28.9% best cells; SP-LLM 48.9% S8 single-cell high; Codex SP-LLM 100% S5 only perfect); ablation BER by configuration; determinism via pytest", "Failure_Modes": "End-to-end bottleneck is code generation not extraction (BSG recall 92.3% vs BER 23.0%, 67-69pp gap); feedback regresses some scenarios (S2 75.8% no-feedback to 12.1% with feedback); no single method dominates all 8; 5 cells 0% BER non-executable; 20/xx runs variance sigma 0.0 largest 48.1", "Threats_Validity": "Internal: BER depends on test-generation prompt though executed deterministically; External: 8 synthetic scenarios only, telecom/banking skew; Construct: BER equals BRPS per-scenario numerically, commercial-tool BER unpublished so no industrial baseline", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "8.0", "BID": "B1-124", "Decision": "INCLUDE-L2", "Role": "supporting (legacy modernization, BSG)", "Extraction_Level": "fulltext", "txt_path": "papers/B1-124.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint" ] }, { "Paper_ID": "S37", "CiteKey": "li2026solvita", "Title": "Solvita: Enhancing Large Language Models for Competitive Programming via Agentic Evolution", "Authors_Year": "Han Li et al. (2026; 13 authors)", "Venue_Tier": "arXiv", "DOI_URL": "http://arxiv.org/abs/2605.15301v1", "Core_Problem": "Statelessness of current multi-agent code generation systems that rely on static retrieval and discard problem-solving and debugging experience gained from previous tasks, struggling on rigorous reasoning in competitive programming.", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "Closed-loop agentic evolution framework comprising 4 specialized agents (Planner, Solver, Oracle, Hacker) where each agent is paired with a trainable, graph-structured knowledge network continuously updated via reinforcement learning (REINFORCE) from outcome signals.", "Coordination_Control": "Closed-loop multi-agent coordination with shared event bus; Planner decomposes formal specs and selects strategy; Solver synthesizes/patches programs; Oracle executes certified test supervision; Hacker generates targeted adversarial test cases and injects contrastive updates to other agents.", "Foundation_Model": "GPT-5.4, Claude Opus 4.6, Qwen3.6, DeepSeek V4 Pro, Grok", "Benchmark_Dataset": "CodeContests (165 problems), APPS (1,000 sampled problems), AetherCode (400 problems), and live Codeforces rounds", "Artifact_Type": "Solvita framework (agentic evolution pipeline and graph knowledge networks)", "Primary_Metrics_Exact": "Pass@1 (%): CodeContests: 82.42% (GPT-5.4), 80.61% (Claude Opus 4.6), 69.70% (Qwen3.6), 89.09% (DeepSeek V4 Pro), 78.18% (Grok); APPS: 67.70% (GPT-5.4), 69.30% (Claude Opus 4.6), 55.10% (Qwen3.6), 68.10% (DeepSeek V4 Pro), 58.50% (Grok); AetherCode: 49.25% (GPT-5.4), 53.75% (Claude Opus 4.6), 26.00% (Qwen3.6), 51.50% (DeepSeek V4 Pro), 33.50% (Grok)", "Secondary_Metrics": "Single-pass baseline pass@1: 40.00% (CC GPT-5.4); Token footprint comparable to open-source agent frameworks (staying far below commercial CLI agents); reduction in algorithmic, edge/spec, TLE, MLE, and runtime errors", "Failure_Modes": "Patch-repair drift where Solver mislabels systemic flaws as localized edits; Hacker scope failure on deep number-theoretic / combinatorial invariants due to LLM reasoning horizon limits; cold-start computational overhead requiring ~5,000 problems to amortize network weights.", "Threats_Validity": "Evaluated on algorithmic/competitive programming benchmarks rather than full repository-level enterprise multi-file software engineering tasks; reliance on synthetic/live competition problem distributions; knowledge graph saturation over prolonged execution.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "8.5", "BID": "B1-125", "Decision": "INCLUDE-L2", "Role": "primary (agentic evolution, competitive programming)", "Extraction_Level": "fulltext", "txt_path": "papers/B1-125.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "coordination" ] }, { "Paper_ID": "S38", "CiteKey": "apostolou2026agentic", "Title": "Agentic AI in Industry: Adoption Level and Deployment Barriers", "Authors_Year": "Spyridon Alvanakis Apostolou, Jan Bosch, Helena Holmstrom Olsson (2026)", "Venue_Tier": "arXiv", "DOI_URL": "http://arxiv.org/abs/2605.14675v1", "Core_Problem": "Sparse empirical evidence on how industrial software engineering organizations adopt agentic AI and what specific organizational and technical barriers prevent deployment into active production SDLC workflows.", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "Qualitative empirical interview study using a six-level maturity framework (Level 0 Non-supported to Level 5 Autonomous Self-Healing) characterizing industrial agentic AI adoption.", "Coordination_Control": "Human-in-the-Loop (HITL) as the sole trusted verification mechanism across industrial workflows; multi-agent orchestration examined at Level 3 maturity.", "Foundation_Model": "Claude Code, GitHub Copilot, Qwen3-14B, gpt-oss", "Benchmark_Dataset": "Qualitative semi-structured interviews with 16 software practitioners across 12 industrial companies of varying sizes and domains (regulated safety-critical, finance, enterprise software)", "Artifact_Type": "Maturity assessment framework and empirical taxonomy of deployment barriers (information asymmetry, qualification absence)", "Primary_Metrics_Exact": "Adoption distribution: 7 companies at Level 1 (AI Assistants), 4 companies at Level 2 (AI Compensators), 1 company at Level 3 (Multi-Agent Orchestration / Superchargers), 0 at Levels 0, 4, 5", "Secondary_Metrics": "4 companies (C6, C7, C8, C12) demonstrated higher-level experimental multi-agent capabilities blocked from production deployment due to verification gaps", "Failure_Modes": "Capability-deployment verification gap caused by 4 recurring barriers: LLM context window constraints during knowledge aggregation, under-performance on proprietary languages/protocols, non-determinism violating qualification standards, and data confidentiality/privacy concerns.", "Threats_Validity": "Internal validity: self-reported practitioner assessments mitigated by adaptive routing questions; Construct validity: adapted maturity framework; External validity: sample of 16 practitioners across 12 European/international companies.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "8.5", "BID": "B1-126", "Decision": "INCLUDE-L2", "Role": "supporting (industrial adoption survey, RQ4)", "Extraction_Level": "fulltext", "txt_path": "papers/B1-126.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint", "hitl", "orchestration" ] }, { "Paper_ID": "S39", "CiteKey": "lwin2026deterministic", "Title": "Deterministic vs. LLM-Controlled Orchestration for COBOL-to-Python Modernization", "Authors_Year": "Naing Oo Lwin, Rajesh Kumar (2026)", "Venue_Tier": "arXiv", "DOI_URL": "http://arxiv.org/abs/2605.09894v1", "Core_Problem": "Unclear trade-offs between delegating execution control to LLMs (agentic orchestration) vs. deterministic fixed-policy orchestration in structured software modernization workflows regarding functional correctness, robustness, and token efficiency.", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "ATLAS modernization framework implementing both deterministic pipeline orchestration (fixed topological execution with explicit validation) and LLM-controlled agentic orchestration (dynamic tool selection and iterative retry), isolating execution control as the sole experimental variable.", "Coordination_Control": "Deterministic fixed-policy pipeline vs. dynamic LLM-directed autonomous agent tool calling with cyclic self-healing / tool-feedback loops.", "Foundation_Model": "Claude-Sonnet-4-5, GPT-5.1-Codex-Max, Grok-Code-Fast-1", "Benchmark_Dataset": "NIST COBOL85 benchmark suite (382 programs, 367 executed across 13 modules including NC, SQ, ST, IF, RL, DB)", "Artifact_Type": "ATLAS framework and benchmark harness (https://github.com/astrio-ai/forall)", "Primary_Metrics_Exact": "Computational Accuracy (CA): Deterministic 0.966 vs LLM-controlled 0.964 (Claude-Sonnet-4-5); 0.969 vs 0.964 (GPT-5.1-Codex-Max); 0.961 vs 0.958 (Grok-Code-Fast-1). Success Rate (SR): Deterministic 0.902 vs LLM-controlled 0.918 (Claude); 0.910 vs 0.937 (GPT); 0.872 vs 0.906 (Grok). P5-CA (5th percentile): Deterministic 0.959 vs LLM 0.956 (Claude); 0.962 vs 0.958 (GPT); 0.951 vs 0.941 (Grok). CVaR0.1: Deterministic 0.953 vs LLM 0.949 (Claude); 0.956 vs 0.951 (GPT); 0.943 vs 0.934 (Grok).", "Secondary_Metrics": "Token consumption: LLM-controlled requires 1.75M - 2.25M tokens per complex module vs. deterministic requiring 400K - 700K tokens (up to 3.5x token reduction under deterministic orchestration); NIST passed tests: 8,886 (COBOL baseline), 9,006 (Python deterministic), 8,986 (Python LLM-controlled)", "Failure_Modes": "Agentic drift and unbounded reasoning loops in LLM-controlled systems causing high token variance and heavier performance tails; deterministic orchestration cannot adapt when unforeseen translation edge-cases arise outside fixed policy rules.", "Threats_Validity": "Evaluated exclusively on legacy COBOL-to-Python translation; reliance on current API pricing models; synthetic and standardized NIST test harness may not fully represent multi-tier enterprise systems.", "Provenance_Tag": "FAKTA-DOK (txt/B1-128.txt Table 1 Sec4-5 CA/SR/P5-CA/CVaR0.1; Code: github.com/astrio-ai/forall)", "QA_Score": "9.5", "BID": "B1-128", "Decision": "INCLUDE-L2", "Role": "primary (deterministic vs LLM orchestration)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-128.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "orchestration" ] }, { "Paper_ID": "S40", "CiteKey": "talluri2026retrieval", "Title": "Retrieval-Conditioned Topology Selection with Provable Budget Conservation for Multi-Agent Code Generation", "Authors_Year": "Abhijit Talluri et al. (2026; 4 authors)", "Venue_Tier": "arXiv", "DOI_URL": "http://arxiv.org/abs/2605.05657v1", "Core_Problem": "Multi-agent LLM systems select orchestration topologies (Fast Path, Subagent, Multi-Agent, Deep Research) without inspecting codebase structural complexity, leading to severe misrouting, execution failure, and unbounded token/resource consumption.", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "Retrieval-Guided Adaptive Orchestration (RGAO) within CODE-AGENT: extracts structural complexity vectors from hierarchical code index, routes topology, and executes sub-agents governed by formal contracts with six-dimensional budget vectors.", "Coordination_Control": "Dynamic hierarchical multi-agent orchestration with provable budget algebra; structural induction conservation theorem (Theorem 1) statically verified in O(|V| + |E|) preventing child delegations from exceeding parent budget bounds.", "Foundation_Model": "Claude Opus 4.5, Claude-3.5-Sonnet-2024-10-22, GPT-5.3, GPT-5.2, GPT-4o", "Benchmark_Dataset": "250-instance routing evaluation set, synthetic SWE-bench proxy harness (10 issues), and microbenchmarks over n=20 pytest-benchmark rounds", "Artifact_Type": "CODE-AGENT framework and RGAO router implementation", "Primary_Metrics_Exact": "Misrouting reduction: from 30.1% (95% CI [26.4, 34.1]) under regex baseline to 8.2% (95% CI [6.1, 10.9]) under RGAO (paired McNemar p < 10^-6); SWE-bench proxy resolution: 40% pass@1 at ~6,000 tokens mean cost vs monolithic agents (50k-120k tokens, 8-20x reduction)", "Secondary_Metrics": "Contract factory instantiation latency: ~1.1 µs median; sub-millisecond DAG construction; linear tree-index retrieval scalability", "Failure_Modes": "Budget conservation guarantees hold strictly under deterministic tool costs, bounded retrieval depth, and finite actions; under stochastic environments or non-zero temperatures, guarantees degrade to expected bounds; routing classifier uses hand-tuned thresholds vulnerable to distribution shift.", "Threats_Validity": "Small synthetic SWE-bench proxy harness (10 issues) due to contamination concerns with official SWE-bench Verified; hand-tuned heuristic routing thresholds rather than fully trained neural routing policy.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "8.5", "BID": "B1-129", "Decision": "INCLUDE-L2", "Role": "primary (retrieval-conditioned topology, RGAO)", "Extraction_Level": "fulltext", "txt_path": "papers/B1-129.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "orchestration" ] }, { "Paper_ID": "S41", "CiteKey": "hasanli2026tdd", "Title": "TDD Governance for Multi-Agent Code Generation via Prompt Engineering", "Authors_Year": "Tarlan Hasanli et al. (2026; 6 authors)", "Venue_Tier": "EASE 2026 / arXiv (cs.SE)", "DOI_URL": "http://arxiv.org/abs/2604.26615v1", "Core_Problem": "LLM instability, non-determinism (up to 75.76% discordant test outputs even at temperature 0), and weak adherence to software engineering discipline in unconstrained workflows, where tests are treated merely as auxiliary inputs rather than enforceable process constraints.", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "AI-native TDD layered governance architecture separating model proposal layer (non-authoritative) from deterministic engine governance layer (authoritative); implements Red-Green-Refactor phase ordering, bounded repair loops, schema/policy validation gating, and atomic mutation control.", "Coordination_Control": "Prompt-centric governance encoding a machine-readable TDD manifesto into prompt orchestration; authoritative orchestration engine verifies schema, security policies, and phase consistency before executing file mutations, preventing infinite repair loops and drift.", "Foundation_Model": "General LLMs (evaluates prompt orchestration and model non-determinism across LLMs; references ChatGPT and frontier models)", "Benchmark_Dataset": "Preliminary empirical evaluation on curated TDD manifesto principles distilled from Kent Beck & Robert C. Martin canon; comparative analysis against unconstrained baseline prompting.", "Artifact_Type": "AI-native TDD governance framework, prompt architecture specification, and manifesto constraint rules (5-page EASE 2026 paper)", "Primary_Metrics_Exact": "Preliminary qualitative stability and discipline adherence: explicit phase separation and validation gating eliminate uncontrolled mutation drift and reduce unstable retry cycles compared to unconstrained baseline prompting.", "Secondary_Metrics": "Trade-off analysis between governance strictness (manifesto injection overhead) and prompt compactness/token context consumption.", "Failure_Modes": "Prompt bloat and token context consumption from manifesto constraint injection; bounded autonomy restricting exploratory refactoring; lack of full runtime semantic verification (prompt-level enforcement only).", "Threats_Validity": "Preliminary empirical validation without large repository-scale benchmark runs (e.g. SWE-bench); prompt-level constraints susceptible to model instruction drift; lack of cross-model quantitative statistical testing.", "Provenance_Tag": "FAKTA-DOK (txt/B1-130.txt; Code: github.com/shahbazsiddeeq/TDD-manifesto)", "QA_Score": "7.0", "BID": "B1-130", "Decision": "INCLUDE-L2", "Role": "supporting (TDD governance, EASE short)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-130.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint", "tdd", "security", "orchestration" ] }, { "Paper_ID": "S42", "CiteKey": "peng2026playcoder", "Title": "PlayCoder: Making LLM-Generated GUI Code Playable", "Authors_Year": "Zhiyuan Peng et al. (2026; 6 authors)", "Venue_Tier": "ACM / arXiv (cs.SE)", "DOI_URL": "http://arxiv.org/abs/2604.19742v1", "Core_Problem": "Existing code-generation benchmarks rely on unit tests, which fail to evaluate event-driven, stateful, interactive GUI applications where syntactically correct and executable code exhibits silent behavioral and logical flaws (e.g., collision failures, invalid state transitions).", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "Closed-loop multi-agent framework comprising PlayDeveloper (repository-aware code generation agent) and PlayRefiner (automated program repair / APR agent), coupled with PlayTester (LLM-based autonomous GUI playthrough testing agent) on PlayEval benchmark.", "Coordination_Control": "Closed-loop iterative control: PlayDeveloper writes initial code -> deterministic compilation/unit test check -> PlayTester executes interactive GUI playthrough and detects logic violations -> PlayRefiner applies targeted edits via visual and execution feedback.", "Foundation_Model": "GPT-5-mini, Claude-Sonnet-3.7, Qwen3-Coder, Claude-Sonnet-4, GPT-4o-mini (10 SOTA code LLMs evaluated)", "Benchmark_Dataset": "PlayEval (43 multilingual GUI applications across 6 categories in Python, TypeScript, and JavaScript) and SWE-bench / HumanEval baseline references", "Artifact_Type": "PlayCoder multi-agent framework, PlayEval benchmark dataset, PlayTester agent, and Play@k metric (open source at https://github.com/Tencent/PlayCoder)", "Primary_Metrics_Exact": "Exec@3 / Pass@3 / Play@3 (%): Claude-Sonnet-4 reaches 36.8% Exec@3 and 20.3% Play@3 on Python; PlayCoder with Qwen3-Coder achieves 38.1% Exec@3, 22.0% Pass@3, 18.9% Play@3 (vs baseline DeepCode 17.9% Exec@3, 6.4% Play@3); JavaScript Exec@3 reaches 44.0%, TypeScript 31.0%.", "Secondary_Metrics": "Efficiency@k (Play@k / tokens per function * 10^3): PlayCoder achieves 8.3% Play@1 at 164K tokens (highest efficiency across all baselines); ablation shows APR removal causes 8.5pp drop in Exec@3, and GUI feedback removal causes 4.5pp drop in Play@3.", "Failure_Modes": "High-frame-rate real-time arcade games (>60 FPS physics like Flappy Bird, Snake) fail in 4/43 projects (9%) due to discrete visual polling latency and GPU inference bottlenecks; TypeScript multi-file state transition degradation.", "Threats_Validity": "VLM limitations in detecting fine-grained visual GUI elements; discrete frame polling missing transient bugs; repository retrieval scaling on massive monorepos; limited scope of 43 benchmark apps.", "Provenance_Tag": "FAKTA-DOK (txt/B1-134.txt Sec3-4: PlayEval 43 repos, Play@k; Code: github.com/Tencent/PlayCoder; Zenodo 10.5281/zenodo.15186407)", "QA_Score": "9.5", "BID": "B1-134", "Decision": "INCLUDE-L2", "Role": "primary (closed-loop GUI feedback, PlayCoder)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-134.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint" ] }, { "Paper_ID": "S43", "CiteKey": "spinner2026bonsai", "Title": "BONSAI: A Mixed-Initiative Workspace for Human-AI Co-Development of Visual Analytics Applications", "Authors_Year": "Thilo Spinner et al. (2026; 4 authors)", "Venue_Tier": "arXiv (cs.HC / cs.SE)", "DOI_URL": "http://arxiv.org/abs/2604.19247v1", "Core_Problem": "Developing complex Visual Analytics applications creates a dilemma between brittle monoliths and restrictive frameworks; unconstrained AI code generation causes uncontrolled architectural drift, lack of fine-grained provenance, and unauditable chaos.", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "Modular 4-layer architecture (Hardware, Services, Orchestration, Application) coupled with a 4-phase development workflow (Plan, Design, Monitor, Review); hierarchical multi-agent structure: Nexus (top-level AI orchestrator), Squad Leads (mid-level coordinators), and AI Development Units (ADUs).", "Coordination_Control": "Mixed-initiative hierarchical orchestration with strict Policy Gates, OpenAPI typed contracts (CType), Git branch isolation per ADU, DAG task decomposition, and fine-grained provenance tracking graph; human sets goals while Nexus coordinates worker ADUs.", "Foundation_Model": "Frontier multimodal LLMs (Claude Code, OpenAI Codex / GPT-series with multimodal figure reasoning)", "Benchmark_Dataset": "Two complex longitudinal case studies: UC1 Component Extraction (extracting Semantic Color Mapping pipeline into composable microservices) and UC2 System Reconstruction (reconstructing PODIUM ranking SVM + UI directly from research paper PDF)", "Artifact_Type": "BONSAI mixed-initiative workspace architecture, MCP-based service registry, and multi-agent coordination platform", "Primary_Metrics_Exact": "End-to-end task completion on case studies: SCM pipeline extracted and assembled in 2 ADU iterations via microservice wiring; PODIUM reproduced with multimodal figure parsing and strict interface contract compliance without architectural drift.", "Secondary_Metrics": "ADU status transition tracking, CType schema violation catching at design-time, reuse-first service discovery flywheel reducing implementation effort to orchestration wiring.", "Failure_Modes": "High cognitive learning curve for developers adopting the 4-layer mental model; manual review bottleneck for admitting new services into registry; upfront contract-first friction inhibiting spontaneous exploratory ideation.", "Threats_Validity": "Empirical validation relies on qualitative longitudinal case studies rather than automated execution benchmarks; potential bias in researcher-led case study evaluation; small team setting.", "Provenance_Tag": "FAKTA-DOK", "QA_Score": "8.0", "BID": "B1-135", "Decision": "INCLUDE-L2", "Role": "supporting (mixed-initiative workspace, HITL)", "Extraction_Level": "fulltext", "txt_path": "papers/B1-135.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint", "hitl", "orchestration" ] }, { "Paper_ID": "S44", "CiteKey": "qiao2026cadmas", "Title": "CADMAS-CTX: Contextual Capability Calibration for Multi-Agent Delegation", "Authors_Year": "Qiao Chuhan, 2026", "Venue_Tier": "ACM / arXiv (cs.AI / cs.SE)", "DOI_URL": "http://arxiv.org/abs/2604.17950v1", "Core_Problem": "Multi-agent delegation systems assume agent capabilities are static scalars at the skill level, causing systematic misdelegation and cascading failures when agent performance varies substantially across task contexts (e.g. isolated edits vs chained repo dependencies).", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "Hierarchical contextual capability calibration framework; maintains Beta posterior distributions (alpha, beta) per agent, skill, and coarse context bucket; uses risk-aware LCB scoring (mu - gamma * sqrt(u)) for decentralized task entry and subtask delegation.", "Coordination_Control": "Locally-centralized / decentralized delegation: task entry agent selected via risk-adjusted contextual score; entry agent decomposes task and routes subtasks to peers only when posterior mean exceeds uncertainty penalty, preventing over-delegation.", "Foundation_Model": "GPT-4o, Claude 3.5 Sonnet (heterogeneous multi-agent team)", "Benchmark_Dataset": "GAIA benchmark (165 validation tasks across 3 difficulty levels) and SWE-bench Lite (300 real-world GitHub issues)", "Artifact_Type": "CADMAS-CTX coordination framework, contextual capability profiling algorithm, and contextual bandit regret bounds", "Primary_Metrics_Exact": "GAIA Validation: CADMAS-CTX achieves 0.442 accuracy [0.425, 0.459] vs Static baseline 0.381 and AutoGen 0.354 (non-overlapping 95% CI); SWE-bench Lite: resolve rate improves from 22.3% [20.1, 24.5] to 31.4% [29.2, 33.6], with chained tasks jumping from 8.4% to 24.8%.", "Secondary_Metrics": "API efficiency on GAIA: 5.1 API calls/task vs AutoGen 18.4 calls/task; robust to tagging noise up to 45% error rate (maintains 29.5% SWE-bench resolve); hard misroute rate reduced by 2 orders of magnitude.", "Failure_Modes": "Coarse context bucketing suffers boundary effects when context transitions are non-linear; performance degrades when context tagging noise exceeds 45%; belief divergence across decentralized agents due to local-only calibration.", "Threats_Validity": "Context buckets (12 theoretical, 3-4 active) rely on heuristic proxy features; single author; evaluation confined to GAIA and SWE-bench Lite without multi-repository enterprise suites.", "Provenance_Tag": "FAKTA-DOK (txt/B1-136.txt Table 1-7 Sec3-5 Fig1-6: GAIA 0.442/SWE-Lite 22.3->31.4% 95% CI; no own repo URL)", "QA_Score": "9.0", "BID": "B1-136", "Decision": "INCLUDE-L2", "Role": "primary (contextual delegation, CADMAS-CTX)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-136.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint" ] }, { "Paper_ID": "S45", "CiteKey": "wang2026from", "Title": "From Translation to Superset: Benchmark-Driven Evolution of a Production AI Agent from Rust to Python", "Authors_Year": "Jinhua Wang, Biswa Sengupta (2026)", "Venue_Tier": "arXiv (cs.SE)", "DOI_URL": "http://arxiv.org/abs/2604.11518v1", "Core_Problem": "Cross-language migration of large rapidly evolving production AI coding agents (648K LOC Rust to Python) without behavioral regression or performance degradation.", "Scope": "Repository/function-level SE code generation-repair (derived from benchmark)", "Proposed_Architecture": "LLM-assisted continuous code translation with benchmark-as-objective-function loop; Codex CLI Python port with codex.enhancements layer supporting 30 feature-flagged extensions.", "Coordination_Control": "Continuous diff-translate-test loop; supervisor guardian safety assessment, multi-agent orchestration, HTTP SSE fallback with 429 backoff and API-key rotation.", "Foundation_Model": "GPT-5.4", "Benchmark_Dataset": "SWE-bench Verified (80 tasks subset) and Terminal-Bench", "Artifact_Type": "Production coding agent Python port (41K LOC, 28 modules), benchmark adapters, prediction artifacts, and test suite.", "Primary_Metrics_Exact": "SWE-bench Verified: 59/80 resolved (73.8%) vs Rust original 56/80 (70.0%); Terminal-Bench: 42.5% accuracy vs Rust original 47.5% (post-fix complete rerun).", "Secondary_Metrics": "15.9x code reduction (648K LOC in Rust to 41K LOC in Python); 90% rank-A cyclomatic complexity; Python local runtime overhead <25ms per tool (<0.1% total session latency); 30 feature-flagged extensions.", "Failure_Modes": "API protocol mismatch (sending local_shell instead of function tools causing 100% fallback); environment pollution via system pip install; silent WebSocket empty-response failure mode; API 400 crashes under malformed payloads; safety-refusal failure mode.", "Threats_Validity": "Sample size limited to 80 SWE-bench Verified tasks; LLM non-determinism accounting for 3.8% delta; reliance on Docker container isolation for pollution detection; benchmark overfitting during iterative refinement.", "Provenance_Tag": "FAKTA-DOK (txt/B1-139.txt Fig1/3-5 SWE-bench 59/80 + Terminal-Bench 42.5%; proprietary, no public URL)", "QA_Score": "9.0", "BID": "B1-139", "Decision": "INCLUDE-L2", "Role": "primary (production code translation, benchmark-driven)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-139.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "orchestration" ] }, { "Paper_ID": "S46", "CiteKey": "melegati2026test", "Title": "Test-Oriented Programming: rethinking coding for the GenAI era", "Authors_Year": "Jorge Melegati, 2026", "Venue_Tier": "arXiv (cs.SE)", "DOI_URL": "http://arxiv.org/abs/2604.08102v1", "Core_Problem": "Developer cognitive overload from reviewing AI-generated production code; proposed elevating developer abstraction to verifying natural-language acceptance test code (Test-Oriented Programming).", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "Onion proof-of-concept CLI tool: parses YAML project specifications, generates YAML class/method architecture, synthesizes acceptance & unit test code, and iteratively generates production code.", "Coordination_Control": "Human-in-the-loop iterative verification: developer reviews/modifies test code and class structures; LLM iterates production code synthesis driven by test execution feedback until all tests pass.", "Foundation_Model": "GPT-4o-mini and Gemini 2.5-Flash", "Benchmark_Dataset": "BibTeX CLI manager tool generation evaluated across 5 runs per model (10 runs total from scratch).", "Artifact_Type": "Open-source CLI tool Onion (GitHub: TOProgramming/onion) and replication package (Zenodo: 10.5281/zenodo.17227298).", "Primary_Metrics_Exact": "10/10 runs successful (100% completion across 5 runs each for GPT-4o-mini and Gemini 2.5-Flash); 0 direct modifications to production code required across all runs.", "Secondary_Metrics": "Intervention frequency on test code (1 edit on GPT-4o-mini vs frequent import/execution fixes on Gemini 2.5-Flash); 2 runs required test comment prompts to guide code generation; code verbosity differences.", "Failure_Modes": "LLM non-determinism across runs generating divergent code; test code inconsistencies/syntax errors causing production generation failure; verbose reasoning model outputs overwhelming reviewers; inability of models to resolve test failures without prompt hints in tests.", "Threats_Validity": "Toy subject system (small BibTeX CLI); single developer evaluator introducing subjective bias; prompts tuned primarily for GPT-4o-mini; scalability to large complex codebases unverified.", "Provenance_Tag": "FAKTA-DOK (txt/B1-141.txt Sec3; Code: github.com/TOProgramming/onion; Zenodo 10.5281/zenodo.17227298)", "QA_Score": "8.0", "BID": "B1-141", "Decision": "INCLUDE-L2", "Role": "supporting (test-oriented programming, vision)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-141.txt", "year_derived": 2026, "tags_derived": [ "supporting", "fulltext", "preprint" ] }, { "Paper_ID": "S47", "CiteKey": "chen2026multi", "Title": "A Multi-Agent Framework for Automated Exploit Generation with Constraint-Guided Comprehension and Reflection", "Authors_Year": "Siyi Chen et al. (2026; 7 authors listed)", "Venue_Tier": "arXiv (cs.SE)", "DOI_URL": "http://arxiv.org/abs/2604.05130v1", "Core_Problem": "Automated Exploit Generation (AEG) in modern open-source software to confirm static analysis vulnerabilities, overcoming constraint-solving limits of SMT/fuzzing and context/hallucination limits of single LLMs.", "Scope": "SE automation scope per benchmark (derived)", "Proposed_Architecture": "VulnSage multi-agent framework: Code Analyzer Agent (taint analysis), Code Generation Agent, Validation Agent (isolated sandbox execution), and Reflection Agents, orchestrated by a central Supervisor Agent.", "Coordination_Control": "Supervisor-orchestrated multi-agent iterative cycles with feedback-driven self-refinement; execution trace and runtime error reflection loop for exploit repair or false positive alert reasoning.", "Foundation_Model": "Qwen3-Max (default); evaluated against Qwen3-Plus, GPT-4o, and DeepSeek-V3", "Benchmark_Dataset": "SecBench.js (JavaScript vulnerability benchmark) and real-world OSS repositories (Java and JavaScript).", "Artifact_Type": "Multi-agent framework implementation, exploit validation harness, and verified reports for 146 zero-day vulnerabilities.", "Primary_Metrics_Exact": "Exploit generation success rate on SecBench.js: 53.47% (34.64% relative improvement over state-of-the-art EXPLOADE.js); 146 zero-day vulnerabilities discovered and verified in real-world OSS.", "Secondary_Metrics": "Token cost ~$0.00082 per 1k input tokens on Qwen3-Max ($0.20 per task on Qwen3-Plus); evaluation across 5 vulnerability types (Prototype Pollution, Command Injection, SQLi, Path Traversal, XSS) in JS and Java.", "Failure_Modes": "LLM hallucination and attention degradation on long traces; LLM memorization risk on public benchmarks; static analyzer alert quality variation; complex higher-order constraint solving failures.", "Threats_Validity": "Evaluated on 5 vulnerability types and 2 languages (JS/Java); lack of standardized ground-truth benchmark for Java AEG; potential LLM data contamination on SecBench.js (mitigated by zero-day real-world evaluation).", "Provenance_Tag": "FAKTA-DOK (txt/B1-142.txt pp6-10 Tables1-3/6 Sec4-5; Code: github.com/Vulnsage/VulnSage)", "QA_Score": "9.5", "BID": "B1-142", "Decision": "INCLUDE-L2", "Role": "primary (exploit generation, VulnSage)", "Extraction_Level": "fulltext", "txt_path": "txt/B1-142.txt", "year_derived": 2026, "tags_derived": [ "primary", "fulltext", "preprint", "security" ] } ]