import numpy as np
from collections import Counter
class DSMEvaluator:
"""Black-box evaluation framework for LLM-generated DSMs."""
def __init__(self, weights=None):
self.weights = weights or {
'completeness': 0.25,
'correctness': 0.25,
'selective_accuracy': 0.20,
'stability': 0.15,
'agreement': 0.15
}
def _to_binary_matrix(self, dsm, elements):
"""Convert DSM dict to binary adjacency matrix."""
n = len(elements)
idx = {e: i for i, e in enumerate(elements)}
mat = np.zeros((n, n))
for (src, tgt), val in dsm.items():
if src in idx and tgt in idx:
mat[idx[src], idx[tgt]] = 1
return mat
def completeness(self, gen_dsm, gt_dsm):
"""Fraction of GT dependencies recovered."""
gen_set = set(gen_dsm.keys())
gt_set = set(gt_dsm.keys())
intersection = gen_set & gt_set
return len(intersection) / max(len(gt_set), 1)
def correctness(self, gen_dsm, gt_dsm):
"""Fraction of GEN dependencies that are correct."""
gen_set = set(gen_dsm.keys())
gt_set = set(gt_dsm.keys())
intersection = gen_set & gt_set
return len(intersection) / max(len(gen_set), 1)
def coupling_density(self, dsm, n_elements):
"""Ratio of actual to possible dependencies."""
n_possible = n_elements * (n_elements - 1)
return len(dsm) / max(n_possible, 1)
def selective_accuracy(self, predictions, ground_truth, abstentions):
"""Accuracy excluding abstained predictions."""
non_abstained = [(p, g) for p, g, a in zip(predictions, ground_truth, abstentions) if not a]
if not non_abstained:
return 0.0
correct = sum(1 for p, g in non_abstained if p == g)
return correct / len(non_abstained)
def abstention_coverage(self, abstentions, ground_truth):
"""Fraction of cases where abstention was correct."""
if not any(abstentions):
return 1.0
return sum(abstentions) / len(abstentions)
def stability_entropy(self, dsm_runs, elements):
"""Entropy across multiple DSM generation runs."""
n = len(elements)
prob_matrix = np.zeros((n, n))
for dsm in dsm_runs:
prob_matrix += self._to_binary_matrix(dsm, elements)
prob_matrix /= max(len(dsm_runs), 1)
probs = prob_matrix.flatten()
probs = probs[probs > 0]
entropy = -np.sum(probs * np.log2(probs + 1e-10))
max_entropy = np.log2(n * n)
return entropy / max(max_entropy, 1)
def fleiss_kappa(self, dsm_runs, elements):
"""Fleiss' kappa for inter-rater agreement across runs."""
n_elements = len(elements)
n_raters = len(dsm_runs)
ratings = np.zeros((n_elements * (n_elements - 1), n_raters))
for r, dsm in enumerate(dsm_runs):
mat = self._to_binary_matrix(dsm, elements)
idx = 0
for i in range(n_elements):
for j in range(n_elements):
if i != j:
ratings[idx, r] = mat[i, j]
idx += 1
n_items, n_raters = ratings.shape
p_j = np.sum(ratings, axis=0) / (n_items * n_raters)
p_bar_e = np.sum(p_j * (1 - p_j))
p_i = np.sum(ratings * (1 - ratings), axis=1) / (n_raters * (n_raters - 1))
p_bar = 1 - np.mean(p_i)
if p_bar_e == 1:
return 1.0
return (p_bar - p_bar_e) / (1 - p_bar_e)
def composite_quality(self, gen_dsm, gt_dsm, dsm_runs, elements,
predictions=None, ground_truth=None, abstentions=None):
"""Compute Composite Quality Score Q."""
metrics = {
'completeness': self.completeness(gen_dsm, gt_dsm),
'correctness': self.correctness(gen_dsm, gt_dsm),
}
if predictions is not None and abstentions is not None:
metrics['selective_accuracy'] = self.selective_accuracy(
predictions, ground_truth, abstentions)
metrics['stability'] = 1 - self.stability_entropy(dsm_runs, elements)
metrics['agreement'] = max(0, self.fleiss_kappa(dsm_runs, elements))
q = sum(self.weights.get(k, 0) * metrics.get(k, 0) for k in self.weights)
return q, metrics
DSM generation, design structure matrix, auto-dsm, MBSE evaluation, LLM systems engineering, model-based decomposition, system architecture evaluation, dependency matrix, systems engineering LLM
