Split Sensitivity Analysis By Performance Central Tendency

import numpy as np
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import make_scorer
from scipy.stats import mannwhitneyu

def split_ratio_sensitivity_test(X, y, split_ratios, model, metric, k=5, num_trials=20, alpha=0.05):
    """
    Evaluates the impact of train/test split ratios on model performance by applying k-fold cross-validation
    to both train and test sets and comparing their central tendencies using a non-parametric test.

    Parameters:
        X (np.ndarray): Feature dataset (numerical variables only).
        y (np.ndarray): Target array.
        split_ratios (list of tuples): Train/test split ratios (e.g., [(0.7, 0.3), (0.8, 0.2)]).
        model: A scikit-learn compatible model instance.
        metric: A callable metric function (e.g., sklearn.metrics.accuracy_score).
        k (int): Number of folds for cross-validation.
        num_trials (int): Number of random trials for each split ratio.
        alpha (float): Significance level for the statistical test (default is 0.05).

    Returns:
        dict: A dictionary summarizing the percentage of trials with equivalent central tendencies for each split ratio.
    """
    results = {}
    scorer = make_scorer(metric)

    for train_ratio, test_ratio in split_ratios:
        ratio_key = f"Train:{train_ratio}, Test:{test_ratio}"
        results[ratio_key] = []

        for _ in range(num_trials):
            # Split the data
            X_train, X_test, y_train, y_test = train_test_split(
                X, y, train_size=train_ratio, test_size=test_ratio, random_state=None
            )

            # Perform k-fold cross-validation on the train set
            train_cv_scores = cross_val_score(model, X_train, y_train, cv=k, scoring=scorer)

            # Perform k-fold cross-validation on the test set
            test_cv_scores = cross_val_score(model, X_test, y_test, cv=k, scoring=scorer)

            # Compare central tendencies using Mann-Whitney U test
            _, p_value = mannwhitneyu(train_cv_scores, test_cv_scores, alternative='two-sided')

            # Evaluate consistency based on p-value
            results[ratio_key].append(p_value > alpha)  # Consistent if p-value > alpha

        # Summarize results for this split ratio
        results[ratio_key] = {
            "Consistent Trials (%)": np.mean(results[ratio_key]) * 100
        }

    return results

# Demo Usage
if __name__ == "__main__":
    from sklearn.datasets import make_classification
    from sklearn.linear_model import LogisticRegression
    from sklearn.metrics import accuracy_score

    # Generate synthetic dataset
    X, y = make_classification(n_samples=1000, n_features=10, random_state=42)

    # Define split ratios to evaluate
    split_ratios = [(0.5, 0.5), (0.7, 0.3), (0.8, 0.2), (0.9, 0.1)]

    # Initialize model and metric
    model = LogisticRegression()
    metric = accuracy_score

    # Run the sensitivity analysis
    results = split_ratio_sensitivity_test(
        X, y, split_ratios, model, metric, k=10, num_trials=30, alpha=0.01
    )

    # Print results
    for split, details in results.items():
        print(f"Split Ratio {split}:")
        for key, value in details.items():
            print(f"  - {key}: {value:.2f}%")
        print("-" * 40)