Skip to content

xwhy.explainers.tabular

Tabular explainer implementation.

TabularExplainer

Bases: BaseExplainer

Explainer for Tabular models utilizing the SMILE algorithm.

This explainer preserves exact Wasserstein LIME mechanics while integrating with the broader framework architectures.

Source code in src/xwhy/explainers/tabular.py 
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
class TabularExplainer(BaseExplainer):
    """Explainer for Tabular models utilizing the SMILE algorithm.

    This explainer preserves exact Wasserstein LIME mechanics while
    integrating with the broader framework architectures.
    """

    def __init__(
        self,
        model: Any,  # noqa: ANN401
        config: TabularConfig | None = None,
        mode: Literal["classification", "regression"] = "classification",
        num_perturbations: int = 500,
        num_distribution_samples: int = 100,
        local_noise: float = 0.05,
        perturbation_noise: float = 0.4,
        seed: int = 42,
        epsilon: float = 1.0,
        kernel_width: float = 0.2,
        ridge_alpha: float = 1.0,
        min_valid_ratio: float = 0.5,
        distance_type: str | DistanceType = DistanceType.WASSERSTEIN,
        surrogate_type: str | SurrogateType = SurrogateType.LIME,
        use_best_surrogate: bool = True,
        device: str = "cpu",
        validate_normalization: bool = True,
        return_p_value: bool = False,
        n_bootstrap: int = 1000,
    ) -> None:
        """Initialize the Tabular explainer.

        Args:
            model: Trained black-box model with a `predict` method.
            config: Optional configuration object.
            mode: Task type ("classification" or "regression").
            num_perturbations: Number of LIME samples generated.
            num_distribution_samples: Samples per feature distribution.
            local_noise: Noise scale for the local instance neighborhood.
            perturbation_noise: Noise scale for perturbation distributions.
            seed: Random seed for reproducibility.
            epsilon: Numerical stability constant.
            kernel_width: Kernel width for similarity weights.
            ridge_alpha: Ridge regularization strength.
            min_valid_ratio: Minimum proportion of valid (non-NaN/non-infinite)
                perturbation evaluations required for reliable surrogate training.
            distance_type: Distance metric definition.
            surrogate_type: Default surrogate method name.
            use_best_surrogate: Automatically search for the best surrogate.
            seed: Random seed for reproducibility.
            device: Device type name.
            validate_normalization: Whether to warn if the input appears not
                to be normalized.
            return_p_value: Whether to compute statistical significance
                (p-values) for computed distances using bootstrap sampling.
            n_bootstrap: Number of bootstrap iterations for p-value estimation.

        Raises:
            ValueError: If mode is invalid or distance configuration fails.

        """
        distance_type = DistanceType.from_str(distance_type)
        surrogate_type = SurrogateType.from_str(surrogate_type)

        if mode not in ("classification", "regression"):
            raise ValueError("mode must be 'classification' or 'regression'.")

        if config is None:
            config = TabularConfig(
                mode=mode,
                num_perturbations=num_perturbations,
                num_distribution_samples=num_distribution_samples,
                local_noise=local_noise,
                perturbation_noise=perturbation_noise,
                seed=seed,
                epsilon=epsilon,
                kernel_width=kernel_width,
                ridge_alpha=ridge_alpha,
                min_valid_ratio=min_valid_ratio,
                distance_type=distance_type,
                surrogate_type=surrogate_type,
                use_best_surrogate=use_best_surrogate,
                device=device,
                validate_normalization=validate_normalization,
                return_p_value=return_p_value,
                n_bootstrap=n_bootstrap,
            )

        if (
            getattr(config, "use_best_surrogate", True)
            or not config.surrogate_type.is_linear_model  # type: ignore[attr-defined]
        ):
            logger.warning(
                "Using a non-linear surrogate model or enabling 'use_best_surrogate' "
                "can replace a black-box model with another complex model, "
                "sacrificing local interpretability. The scientific community highly "
                "recommends utilizing simple linear models (e.g., LIME, OLS) to "
                "guarantee transparent and additive feature attributions."
            )

        super().__init__(config)
        self.state = TabularState()
        self.state.model = TabularModelAdapter(
            model=model, device=getattr(config, "device", "cpu")
        )

        self._rng = np.random.default_rng(self.config.seed)  # type: ignore[union-attr]

    def _generate_instance_distribution(
        self, instance: np.ndarray, num_features: int, noise: float, samples: int
    ) -> np.ndarray:
        """Create local Gaussian distributions for each feature.

        Args:
            instance: Target instance to explain.
            num_features: Number of features.
            noise: Variance parameter for normal distribution.
            samples: Number of observations per distribution.

        Returns:
            np.ndarray: Matrix of the local distribution.

        """
        distribution = np.zeros((samples, num_features))
        for i in range(num_features):
            distribution[:, i] = instance[i] + self._rng.normal(0, noise, samples)
        return distribution

    def explain(
        self,
        instance: np.ndarray | Sequence[Any],
        feature_names: Sequence[str] | None = None,
        fidelity_plot: bool = False,
        **kwargs: Any,  # noqa: ANN401
    ) -> TabularXWhyResult:
        """Generate an explanation using the specified distance algorithm.

        Args:
            instance: Target instance array of shape [n_features].
            feature_names: Optional sequence specifying column names.
            fidelity_plot: Rendering fidelity scatter plot.
            **kwargs: Additional parameters.

        Returns:
            TabularXWhyResult: The structured outcome containing weights,
                distances, and surrogate coefficients.

        Raises:
            ValueError: If the instance contains out-of-scale values, indicating
                a lack of standardization.

        """
        cfg: TabularConfig = self.config  # type: ignore[assignment]
        instance_arr = np.asarray(instance, dtype=np.float64)

        if cfg.validate_normalization and np.abs(np.mean(instance_arr)) > 5.0:
            logger.warning(
                "Instance appears not normalized. Ensure you pass standardized data."
            )

        num_features = len(instance_arr)

        # 1. Generate base perturbation samples
        x_matrix = self._rng.normal(0, 1, size=(cfg.num_perturbations, num_features))

        # 2. Local distribution around original instance
        instance_dist = self._generate_instance_distribution(
            instance=instance_arr,
            num_features=num_features,
            noise=cfg.local_noise,
            samples=cfg.num_distribution_samples,
        )

        y_target = np.zeros((cfg.num_perturbations,))
        distances = np.zeros((cfg.num_perturbations,))

        return_p_val = getattr(cfg, "return_p_value", False)
        n_bootstrap = getattr(cfg, "n_bootstrap", 1000)

        p_values_matrix = (
            np.zeros((cfg.num_perturbations, num_features)) if return_p_val else None
        )

        logger.info(
            "Computing distances for %d perturbations...", cfg.num_perturbations
        )

        # 3. Main Loop
        for idx, sample in enumerate(x_matrix):
            sample_dist = self._generate_instance_distribution(
                instance=sample,
                num_features=num_features,
                noise=cfg.perturbation_noise,
                samples=cfg.num_distribution_samples,
            )

            preds = self.state.model.predict(sample_dist)  # type: ignore[union-attr]

            if cfg.mode == "classification":
                y_target[idx] = np.bincount(preds.astype(int)).argmax()
            else:
                y_target[idx] = np.mean(preds)

            # ==============================
            # Compute distance (per feature)
            # ==============================
            dist_total = 0.0
            for j in range(num_features):
                res = calculate_distance(
                    metric=cfg.distance_type,
                    source=instance_dist[:, j],
                    target=sample_dist[:, j],
                    return_p_value=return_p_val,
                    n_bootstrap=n_bootstrap,
                )

                if isinstance(res, tuple):
                    p_val, dist_val = res
                    if p_values_matrix is not None:
                        p_values_matrix[idx, j] = p_val
                else:
                    dist_val = res

                dist_total += dist_val

            distances[idx] = dist_total

        # ---------------------------------------------------------
        # Distance Validation & Filtering setup:
        # Convert distances to numpy array and drop non-finite (inf/NaN) values.
        # ---------------------------------------------------------
        logger.info("Validating perturbation distances...")
        distances_raw = np.array(distances, dtype=float)

        # Identify valid (non-infinite, non-NaN) distances
        valid_mask = np.isfinite(distances_raw)
        valid_count = int(np.sum(valid_mask))
        total_count = len(distances_raw)
        valid_ratio = valid_count / total_count

        # Check validity threshold and warn if insufficient
        min_valid_ratio = cfg.min_valid_ratio

        if valid_count == 0:
            error_msg = (
                "All perturbations failed (0 valid distances). Cannot fit the "
                "surrogate model with an empty dataset. Aborting explanation."
            )
            logger.error(error_msg)
            raise ValueError(error_msg)
        elif valid_ratio < min_valid_ratio:
            logger.warning(
                "Low valid perturbation ratio. Only %.1f%% succeeded (%d/%d). "
                "Training surrogate model with reduced sample size, which may "
                "lead to unstable explanations.",
                valid_ratio * 100,
                valid_count,
                total_count,
            )

        # Filter arrays to drop failed evaluations cleanly
        x_valid = x_matrix[valid_mask]
        y_valid = y_target[valid_mask]
        distances_valid = distances_raw[valid_mask]

        # Surrogate Training via Framework
        if cfg.use_best_surrogate:
            logger.info("Searching for optimal surrogate model...")
            method, score = SurrogateTrainer.find_best(
                x=x_valid,
                y=y_valid,
                distances=distances_valid,
                seed=cfg.seed,
                kernel_width=cfg.kernel_width,
                epsilon=cfg.epsilon,
                ridge_alpha=cfg.ridge_alpha,
                normalize_distances=False,
            )
            logger.info(
                "Optimization complete. Selected surrogate model:"
                " '%s' (Best Score: %.4f)",
                method.value,
                score,
            )
        else:
            method = cfg.surrogate_type  # type: ignore[assignment]
            logger.info("Skipping surrogate search. Using default: '%s'", method.value)

        weights = SurrogateTrainer.compute_weights(
            method=method,
            distances=distances_valid,
            kernel_width=cfg.kernel_width,
            epsilon=cfg.epsilon,
            normalize_distances=False,
        )

        logger.info("Training surrogate model (%s)...", method.value)
        surrogate = SurrogateFactory.create(
            method=method,
            seed=cfg.seed,
            ridge_alpha=cfg.ridge_alpha,
        )
        surrogate.fit(x_valid, y_valid, weights)

        coeffs = surrogate.coefficients()
        y_pred_valid = surrogate.predict(x_valid)

        metrics = RegressionMetrics.calculate(
            y_true=y_valid,
            y_pred=y_pred_valid,
            weights=weights,
            num_features=len(coeffs),
        )

        if cfg.mode == "classification":
            y_pred_valid = np.round(y_pred_valid).astype(int).flatten()
        else:
            y_pred_valid = y_pred_valid.flatten()

        raw_data: dict[str, Any] = {
            "x_matrix": x_valid,
            "y_target": y_valid,
            "y_pred": y_pred_valid,
            "weights": weights,
            "distances": distances_valid,
            "surrogate_method": method,
        }

        if p_values_matrix is not None:
            raw_data["p_values"] = p_values_matrix[valid_mask]

        result = TabularXWhyResult(
            coefficients=coeffs,
            metrics=metrics,
            raw_data=raw_data,
            instance=instance_arr,
            feature_list=feature_names or [],
            base_values=0.0,
        )

        if fidelity_plot:
            logger.info("Rendering fidelity plot as requested...")
            result.plot(show=True)

        return result

__init__(model, config=None, mode='classification', num_perturbations=500, num_distribution_samples=100, local_noise=0.05, perturbation_noise=0.4, seed=42, epsilon=1.0, kernel_width=0.2, ridge_alpha=1.0, min_valid_ratio=0.5, distance_type=DistanceType.WASSERSTEIN, surrogate_type=SurrogateType.LIME, use_best_surrogate=True, device='cpu', validate_normalization=True, return_p_value=False, n_bootstrap=1000)

Initialize the Tabular explainer.

Parameters:

Name Type Description Default
model Any

Trained black-box model with a predict method.

 required
config TabularConfig | None

Optional configuration object.

 None
mode Literal['classification', 'regression']

Task type ("classification" or "regression").

 'classification'
num_perturbations int

Number of LIME samples generated.

 500
num_distribution_samples int

Samples per feature distribution.

 100
local_noise float

Noise scale for the local instance neighborhood.

 0.05
perturbation_noise float

Noise scale for perturbation distributions.

 0.4
seed int

Random seed for reproducibility.

 42
epsilon float

Numerical stability constant.

 1.0
kernel_width float

Kernel width for similarity weights.

 0.2
ridge_alpha float

Ridge regularization strength.

 1.0
min_valid_ratio float

Minimum proportion of valid (non-NaN/non-infinite) perturbation evaluations required for reliable surrogate training.

 0.5
distance_type str | DistanceType

Distance metric definition.

 WASSERSTEIN
surrogate_type str | SurrogateType

Default surrogate method name.

 LIME
use_best_surrogate bool

Automatically search for the best surrogate.

 True
seed int

Random seed for reproducibility.

 42
device str

Device type name.

 'cpu'
validate_normalization bool

Whether to warn if the input appears not to be normalized.

 True
return_p_value bool

Whether to compute statistical significance (p-values) for computed distances using bootstrap sampling.

 False
n_bootstrap int

Number of bootstrap iterations for p-value estimation.

 1000

Raises:

Type Description
ValueError

If mode is invalid or distance configuration fails.
Source code in src/xwhy/explainers/tabular.py 
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
def __init__(
    self,
    model: Any,  # noqa: ANN401
    config: TabularConfig | None = None,
    mode: Literal["classification", "regression"] = "classification",
    num_perturbations: int = 500,
    num_distribution_samples: int = 100,
    local_noise: float = 0.05,
    perturbation_noise: float = 0.4,
    seed: int = 42,
    epsilon: float = 1.0,
    kernel_width: float = 0.2,
    ridge_alpha: float = 1.0,
    min_valid_ratio: float = 0.5,
    distance_type: str | DistanceType = DistanceType.WASSERSTEIN,
    surrogate_type: str | SurrogateType = SurrogateType.LIME,
    use_best_surrogate: bool = True,
    device: str = "cpu",
    validate_normalization: bool = True,
    return_p_value: bool = False,
    n_bootstrap: int = 1000,
) -> None:
    """Initialize the Tabular explainer.

    Args:
        model: Trained black-box model with a `predict` method.
        config: Optional configuration object.
        mode: Task type ("classification" or "regression").
        num_perturbations: Number of LIME samples generated.
        num_distribution_samples: Samples per feature distribution.
        local_noise: Noise scale for the local instance neighborhood.
        perturbation_noise: Noise scale for perturbation distributions.
        seed: Random seed for reproducibility.
        epsilon: Numerical stability constant.
        kernel_width: Kernel width for similarity weights.
        ridge_alpha: Ridge regularization strength.
        min_valid_ratio: Minimum proportion of valid (non-NaN/non-infinite)
            perturbation evaluations required for reliable surrogate training.
        distance_type: Distance metric definition.
        surrogate_type: Default surrogate method name.
        use_best_surrogate: Automatically search for the best surrogate.
        seed: Random seed for reproducibility.
        device: Device type name.
        validate_normalization: Whether to warn if the input appears not
            to be normalized.
        return_p_value: Whether to compute statistical significance
            (p-values) for computed distances using bootstrap sampling.
        n_bootstrap: Number of bootstrap iterations for p-value estimation.

    Raises:
        ValueError: If mode is invalid or distance configuration fails.

    """
    distance_type = DistanceType.from_str(distance_type)
    surrogate_type = SurrogateType.from_str(surrogate_type)

    if mode not in ("classification", "regression"):
        raise ValueError("mode must be 'classification' or 'regression'.")

    if config is None:
        config = TabularConfig(
            mode=mode,
            num_perturbations=num_perturbations,
            num_distribution_samples=num_distribution_samples,
            local_noise=local_noise,
            perturbation_noise=perturbation_noise,
            seed=seed,
            epsilon=epsilon,
            kernel_width=kernel_width,
            ridge_alpha=ridge_alpha,
            min_valid_ratio=min_valid_ratio,
            distance_type=distance_type,
            surrogate_type=surrogate_type,
            use_best_surrogate=use_best_surrogate,
            device=device,
            validate_normalization=validate_normalization,
            return_p_value=return_p_value,
            n_bootstrap=n_bootstrap,
        )

    if (
        getattr(config, "use_best_surrogate", True)
        or not config.surrogate_type.is_linear_model  # type: ignore[attr-defined]
    ):
        logger.warning(
            "Using a non-linear surrogate model or enabling 'use_best_surrogate' "
            "can replace a black-box model with another complex model, "
            "sacrificing local interpretability. The scientific community highly "
            "recommends utilizing simple linear models (e.g., LIME, OLS) to "
            "guarantee transparent and additive feature attributions."
        )

    super().__init__(config)
    self.state = TabularState()
    self.state.model = TabularModelAdapter(
        model=model, device=getattr(config, "device", "cpu")
    )

    self._rng = np.random.default_rng(self.config.seed)  # type: ignore[union-attr]

explain(instance, feature_names=None, fidelity_plot=False, **kwargs)

Generate an explanation using the specified distance algorithm.

Parameters:

Name Type Description Default
instance ndarray | Sequence[Any]

Target instance array of shape [n_features].

 required
feature_names Sequence[str] | None

Optional sequence specifying column names.

 None
fidelity_plot bool

Rendering fidelity scatter plot.

 False
**kwargs Any

Additional parameters.

 {}

Returns:

Name Type Description
TabularXWhyResult TabularXWhyResult

The structured outcome containing weights, distances, and surrogate coefficients.

Raises:

Type Description
ValueError

If the instance contains out-of-scale values, indicating a lack of standardization.
Source code in src/xwhy/explainers/tabular.py 
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
def explain(
    self,
    instance: np.ndarray | Sequence[Any],
    feature_names: Sequence[str] | None = None,
    fidelity_plot: bool = False,
    **kwargs: Any,  # noqa: ANN401
) -> TabularXWhyResult:
    """Generate an explanation using the specified distance algorithm.

    Args:
        instance: Target instance array of shape [n_features].
        feature_names: Optional sequence specifying column names.
        fidelity_plot: Rendering fidelity scatter plot.
        **kwargs: Additional parameters.

    Returns:
        TabularXWhyResult: The structured outcome containing weights,
            distances, and surrogate coefficients.

    Raises:
        ValueError: If the instance contains out-of-scale values, indicating
            a lack of standardization.

    """
    cfg: TabularConfig = self.config  # type: ignore[assignment]
    instance_arr = np.asarray(instance, dtype=np.float64)

    if cfg.validate_normalization and np.abs(np.mean(instance_arr)) > 5.0:
        logger.warning(
            "Instance appears not normalized. Ensure you pass standardized data."
        )

    num_features = len(instance_arr)

    # 1. Generate base perturbation samples
    x_matrix = self._rng.normal(0, 1, size=(cfg.num_perturbations, num_features))

    # 2. Local distribution around original instance
    instance_dist = self._generate_instance_distribution(
        instance=instance_arr,
        num_features=num_features,
        noise=cfg.local_noise,
        samples=cfg.num_distribution_samples,
    )

    y_target = np.zeros((cfg.num_perturbations,))
    distances = np.zeros((cfg.num_perturbations,))

    return_p_val = getattr(cfg, "return_p_value", False)
    n_bootstrap = getattr(cfg, "n_bootstrap", 1000)

    p_values_matrix = (
        np.zeros((cfg.num_perturbations, num_features)) if return_p_val else None
    )

    logger.info(
        "Computing distances for %d perturbations...", cfg.num_perturbations
    )

    # 3. Main Loop
    for idx, sample in enumerate(x_matrix):
        sample_dist = self._generate_instance_distribution(
            instance=sample,
            num_features=num_features,
            noise=cfg.perturbation_noise,
            samples=cfg.num_distribution_samples,
        )

        preds = self.state.model.predict(sample_dist)  # type: ignore[union-attr]

        if cfg.mode == "classification":
            y_target[idx] = np.bincount(preds.astype(int)).argmax()
        else:
            y_target[idx] = np.mean(preds)

        # ==============================
        # Compute distance (per feature)
        # ==============================
        dist_total = 0.0
        for j in range(num_features):
            res = calculate_distance(
                metric=cfg.distance_type,
                source=instance_dist[:, j],
                target=sample_dist[:, j],
                return_p_value=return_p_val,
                n_bootstrap=n_bootstrap,
            )

            if isinstance(res, tuple):
                p_val, dist_val = res
                if p_values_matrix is not None:
                    p_values_matrix[idx, j] = p_val
            else:
                dist_val = res

            dist_total += dist_val

        distances[idx] = dist_total

    # ---------------------------------------------------------
    # Distance Validation & Filtering setup:
    # Convert distances to numpy array and drop non-finite (inf/NaN) values.
    # ---------------------------------------------------------
    logger.info("Validating perturbation distances...")
    distances_raw = np.array(distances, dtype=float)

    # Identify valid (non-infinite, non-NaN) distances
    valid_mask = np.isfinite(distances_raw)
    valid_count = int(np.sum(valid_mask))
    total_count = len(distances_raw)
    valid_ratio = valid_count / total_count

    # Check validity threshold and warn if insufficient
    min_valid_ratio = cfg.min_valid_ratio

    if valid_count == 0:
        error_msg = (
            "All perturbations failed (0 valid distances). Cannot fit the "
            "surrogate model with an empty dataset. Aborting explanation."
        )
        logger.error(error_msg)
        raise ValueError(error_msg)
    elif valid_ratio < min_valid_ratio:
        logger.warning(
            "Low valid perturbation ratio. Only %.1f%% succeeded (%d/%d). "
            "Training surrogate model with reduced sample size, which may "
            "lead to unstable explanations.",
            valid_ratio * 100,
            valid_count,
            total_count,
        )

    # Filter arrays to drop failed evaluations cleanly
    x_valid = x_matrix[valid_mask]
    y_valid = y_target[valid_mask]
    distances_valid = distances_raw[valid_mask]

    # Surrogate Training via Framework
    if cfg.use_best_surrogate:
        logger.info("Searching for optimal surrogate model...")
        method, score = SurrogateTrainer.find_best(
            x=x_valid,
            y=y_valid,
            distances=distances_valid,
            seed=cfg.seed,
            kernel_width=cfg.kernel_width,
            epsilon=cfg.epsilon,
            ridge_alpha=cfg.ridge_alpha,
            normalize_distances=False,
        )
        logger.info(
            "Optimization complete. Selected surrogate model:"
            " '%s' (Best Score: %.4f)",
            method.value,
            score,
        )
    else:
        method = cfg.surrogate_type  # type: ignore[assignment]
        logger.info("Skipping surrogate search. Using default: '%s'", method.value)

    weights = SurrogateTrainer.compute_weights(
        method=method,
        distances=distances_valid,
        kernel_width=cfg.kernel_width,
        epsilon=cfg.epsilon,
        normalize_distances=False,
    )

    logger.info("Training surrogate model (%s)...", method.value)
    surrogate = SurrogateFactory.create(
        method=method,
        seed=cfg.seed,
        ridge_alpha=cfg.ridge_alpha,
    )
    surrogate.fit(x_valid, y_valid, weights)

    coeffs = surrogate.coefficients()
    y_pred_valid = surrogate.predict(x_valid)

    metrics = RegressionMetrics.calculate(
        y_true=y_valid,
        y_pred=y_pred_valid,
        weights=weights,
        num_features=len(coeffs),
    )

    if cfg.mode == "classification":
        y_pred_valid = np.round(y_pred_valid).astype(int).flatten()
    else:
        y_pred_valid = y_pred_valid.flatten()

    raw_data: dict[str, Any] = {
        "x_matrix": x_valid,
        "y_target": y_valid,
        "y_pred": y_pred_valid,
        "weights": weights,
        "distances": distances_valid,
        "surrogate_method": method,
    }

    if p_values_matrix is not None:
        raw_data["p_values"] = p_values_matrix[valid_mask]

    result = TabularXWhyResult(
        coefficients=coeffs,
        metrics=metrics,
        raw_data=raw_data,
        instance=instance_arr,
        feature_list=feature_names or [],
        base_values=0.0,
    )

    if fidelity_plot:
        logger.info("Rendering fidelity plot as requested...")
        result.plot(show=True)

    return result