Modeling_Tool.Core

Foundation layer: binning, model registry, sample-weight helpers, the ODPS client, the parallel apply engine, ProcCompare, encryption, JSON conversion, slope computation, and general utilities. The other subpackages build on it. At import time it imports none of them (see the import rules).

Sample Weights: sample_weight_utils

Resolves weight_col / sample_weight (and the wgt_col / wgt aliases) for uniform use by the Model and Eval layers.

sample_weight_utils

Sample-weight resolution and weighted aggregation helpers.

resolve_sample_weight

resolve_sample_weight(data=None, weight_col=None, sample_weight=None, expected_len=None, wgt=None, wgt_col=None)

Resolve sample weights from a DataFrame column or array.

Accepts weight_col / wgt_col and sample_weight / wgt aliases. Returns None when no weight source is provided.

Parameters:

Name	Type	Description	Default
data	DataFrame or None	Frame that holds the weight column; required when weight_col (or wgt_col) is given.	None
weight_col	str or None	Name of the weight column in data.	None
sample_weight	array - like or None	Weights given directly, one per row.	None
expected_len	int or None	Required length of the weight vector; None skips the length check.	None
wgt	array - like or None	Alias of sample_weight, used only when sample_weight is None.	None
wgt_col	str or None	Alias of weight_col, used only when weight_col is None.	None

Returns:

Type	Description
ndarray or None	The validated 1-D float weight vector (see validate_sample_weight), or None when neither a column nor an array was given.

Raises:

Type	Description
ValueError	If both a column and an array are given, if a column is given without data, or if the weights are invalid.
KeyError	If the weight column is not in data.

Source code in Modeling_Tool/Core/sample_weight_utils.py

def resolve_sample_weight(
    data=None,
    weight_col=None,
    sample_weight=None,
    expected_len=None,
    wgt=None,
    wgt_col=None,
):
    """Resolve sample weights from a DataFrame column or array.

    Accepts ``weight_col`` / ``wgt_col`` and ``sample_weight`` / ``wgt`` aliases.
    Returns ``None`` when no weight source is provided.

    Parameters
    ----------
    data : pandas.DataFrame or None, default None
        Frame that holds the weight column; required when ``weight_col`` (or ``wgt_col``) is given.
    weight_col : str or None, default None
        Name of the weight column in ``data``.
    sample_weight : array-like or None, default None
        Weights given directly, one per row.
    expected_len : int or None, default None
        Required length of the weight vector; None skips the length check.
    wgt : array-like or None, default None
        Alias of ``sample_weight``, used only when ``sample_weight`` is None.
    wgt_col : str or None, default None
        Alias of ``weight_col``, used only when ``weight_col`` is None.

    Returns
    -------
    numpy.ndarray or None
        The validated 1-D float weight vector (see ``validate_sample_weight``), or None when neither a column nor an array
        was given.

    Raises
    ------
    ValueError
        If both a column and an array are given, if a column is given without ``data``, or if the weights are invalid.
    KeyError
        If the weight column is not in ``data``.
    """
    if weight_col is None:
        weight_col = wgt_col
    if sample_weight is None:
        sample_weight = wgt

    if sample_weight is not None and weight_col is not None:
        raise ValueError("Provide either weight_col or sample_weight, not both.")

    if weight_col is not None:
        if data is None:
            raise ValueError("data is required when weight_col is provided.")
        if weight_col not in data.columns:
            raise KeyError("weight column '{0}' not found in data".format(weight_col))
        sample_weight = data[weight_col].values

    if sample_weight is None:
        return None

    return validate_sample_weight(sample_weight, expected_len=expected_len)

validate_sample_weight

validate_sample_weight(weights, expected_len=None)

Validate and return a 1-D float numpy weight vector.

Parameters:

Name	Type	Description	Default
weights	array - like	Candidate weights, converted with numpy.asarray(weights, dtype=float).	required
expected_len	int or None	Required length; None skips the length check.	None

Returns:

Type	Description
ndarray	The weights as a 1-D float array.

Raises:

Type	Description
ValueError	If the array is not 1-dimensional, has a length other than expected_len, contains NaN or infinity or a negative value, or sums to zero or less.

Source code in Modeling_Tool/Core/sample_weight_utils.py

def validate_sample_weight(weights, expected_len=None):
    """Validate and return a 1-D float numpy weight vector.

    Parameters
    ----------
    weights : array-like
        Candidate weights, converted with ``numpy.asarray(weights, dtype=float)``.
    expected_len : int or None, default None
        Required length; None skips the length check.

    Returns
    -------
    numpy.ndarray
        The weights as a 1-D float array.

    Raises
    ------
    ValueError
        If the array is not 1-dimensional, has a length other than ``expected_len``, contains NaN or infinity or a
        negative value, or sums to zero or less.
    """
    w = np.asarray(weights, dtype=float)
    if w.ndim != 1:
        raise ValueError("sample_weight must be 1-dimensional.")
    if expected_len is not None and len(w) != expected_len:
        raise ValueError(
            "sample_weight length {0} != expected {1}".format(len(w), expected_len)
        )
    if not np.all(np.isfinite(w)):
        raise ValueError("sample_weight must be finite (no NaN/inf).")
    if np.any(w < 0):
        raise ValueError("sample_weight must be non-negative.")
    total = float(np.sum(w))
    if total <= 0:
        raise ValueError(
            "sample_weight sum must be > 0; got sum={0}. "
            "All-zero weights make weighted metrics undefined.".format(total)
        )
    return w

weighted_sum

weighted_sum(values, weights)

Weighted sum of values.

Parameters:

Name	Type	Description	Default
values	array - like	Values to add up.	required
weights	array - like	Weights, one per value; no validation is done.	required

Returns:

Type	Description
float	sum(values * weights).

Source code in Modeling_Tool/Core/sample_weight_utils.py

def weighted_sum(values, weights):
    """Weighted sum of values.

    Parameters
    ----------
    values : array-like
        Values to add up.
    weights : array-like
        Weights, one per value; no validation is done.

    Returns
    -------
    float
        ``sum(values * weights)``.
    """
    v = np.asarray(values, dtype=float)
    w = np.asarray(weights, dtype=float)
    return float(np.sum(v * w))

weighted_mean

weighted_mean(values, weights)

Weighted mean; returns NaN when total weight is zero.

Parameters:

Name	Type	Description	Default
values	array - like	Values to average.	required
weights	array - like	Weights, one per value; no validation is done.	required

Returns:

Type	Description
float	sum(values * weights) / sum(weights), or NaN when the weights sum to zero.

Source code in Modeling_Tool/Core/sample_weight_utils.py

def weighted_mean(values, weights):
    """Weighted mean; returns NaN when total weight is zero.

    Parameters
    ----------
    values : array-like
        Values to average.
    weights : array-like
        Weights, one per value; no validation is done.

    Returns
    -------
    float
        ``sum(values * weights) / sum(weights)``, or NaN when the weights sum to zero.
    """
    w = np.asarray(weights, dtype=float)
    total = float(np.sum(w))
    if total == 0:
        return np.nan
    return weighted_sum(values, weights) / total

weighted_rate

weighted_rate(mask, weights)

Weighted rate of True / 1 values in mask.

Parameters:

Name	Type	Description	Default
mask	array - like	Boolean (or 0/1) flags, one per row.	required
weights	array - like	Weights, one per row; no validation is done.	required

Returns:

Type	Description
float	The share of the total weight that falls on flagged rows; NaN when the weights sum to zero.

Source code in Modeling_Tool/Core/sample_weight_utils.py

def weighted_rate(mask, weights):
    """Weighted rate of True / 1 values in mask.

    Parameters
    ----------
    mask : array-like
        Boolean (or 0/1) flags, one per row.
    weights : array-like
        Weights, one per row; no validation is done.

    Returns
    -------
    float
        The share of the total weight that falls on flagged rows; NaN when the weights sum to zero.
    """
    m = np.asarray(mask, dtype=float)
    return weighted_mean(m, weights)

Binning: Binning_Tool

Binning_Tool

NumVarBinning

Compute the cut-point sequence of a numeric variable from the number of bins and the binning method.

The cut points are computed from the specified number of bins; the binning methods are "equal-width" and "equal-frequency": (1) When the number of bins is not smaller than the number of unique values of the variable, the cut-point sequence is the sequence of unique values (2) When the number of bins is smaller than the number of unique values, either "equal-width" or "equal-frequency" binning can be chosen

If values that must be binned separately are specified: (1) Add the values to the cut-point sequence (2) Add the upper bound of each value = value + 0.1**(precision+1) to the cut-point sequence

Cut-point precision: (1) If the variable takes integer values, the precision is 1 decimal place (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

For better generalization, the minimum of the cut-point sequence is replaced with -inf and +inf is added as the maximum

Parameters:

Name	Type	Description	Default
var_name	str	Variable name	required
spec_values	list	Special values to be binned separately	None
spec_digit	int	Number of decimal places kept for the upper bound of a special cut point; fixed to 1 if the variable is an integer, otherwise equal to spec_digit	3

Examples:

>>> nvb = NumVarBinning(var_name='income', spec_values=[-9999])
>>> binning = nvb.equi_binning(df, bins=10)

Source code in Modeling_Tool/Core/Binning_Tool.py

class NumVarBinning:
    """
    Compute the cut-point sequence of a numeric variable from the number of bins and the binning method.

    The cut points are computed from the specified number of bins; the binning methods are "equal-width" and "equal-frequency":
    (1) When the number of bins is not smaller than the number of unique values of the variable, the cut-point sequence is the sequence of unique values
    (2) When the number of bins is smaller than the number of unique values, either "equal-width" or "equal-frequency" binning can be chosen

    If values that must be binned separately are specified:
    (1) Add the values to the cut-point sequence
    (2) Add the upper bound of each value = value + 0.1**(precision+1) to the cut-point sequence

    Cut-point precision:
    (1) If the variable takes integer values, the precision is 1 decimal place
    (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

    For better generalization, the minimum of the cut-point sequence is replaced with -inf and +inf is added as the maximum

    Parameters
    ----------
    var_name : str
        Variable name
    spec_values : list, optional
        Special values to be binned separately
    spec_digit : int, default 3
        Number of decimal places kept for the upper bound of a special cut point; fixed to 1 if the variable is an integer, otherwise equal to spec_digit

    Examples
    --------
    >>> nvb = NumVarBinning(var_name='income', spec_values=[-9999])
    >>> binning = nvb.equi_binning(df, bins=10)
    """
    def __init__(self, var_name, spec_values=None, spec_digit=3):
        """
        Initialize the numeric-variable binning object.

        Parameters
        ----------
        var_name : str
            Variable name
        spec_values : list, optional
            Special values to be binned separately
        spec_digit : int, default 3
            Number of decimal places of precision for special values
        """
        self.var_name = var_name
        self.spec_values = spec_values
        self.spec_digit = spec_digit
        self.bins = None
        self.cut_points = None
        self.bin_names = None

    def calc_equi_cutpoints(self, df, bins=10, equi_method="equif"):
        """
        Compute the cut-point sequence of a numeric variable from the number of bins and the equi-binning method.

        1. Compute the cut points from the specified number of bins; the binning methods are "equal-width" and "equal-frequency"
        (1) When the number of bins is not smaller than the number of unique values of the variable, the cut-point sequence is the sequence of unique values
        (2) When the number of bins is smaller than the number of unique values, either "equal-width" or "equal-frequency" binning can be chosen

        2. If values that must be binned separately are specified:
        (1) Add the values to the cut-point sequence;
        (2) Add the upper bound of each value = value + 0.1**(precision+1) to the cut-point sequence;

        3. Cut-point precision:
        (1) If the variable takes integer values, the precision is 1 decimal place
        (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

        Parameters
        ----------
        df : pandas.DataFrame
            Data table
        bins : int, default 10
            Number of bins
        equi_method : string, default "equif"
            Binning method, candidate values {"equid": equal-width, "equif": equal-frequency}

        Returns
        -------
        cut_points : numpy.array
            Cut-point sequence
        """
        var_series = df[self.var_name]

        if any(pd.isnull(var_series)):
            raise ValueError(f"{self.var_name}: the variable contains NaN values")

        if pd.api.types.is_integer_dtype(var_series):
            spec_digit = 1
        else:
            spec_digit = self.spec_digit

        # Compute the cut-point values from the specified number of bins
        cut_points = var_series.unique()
        if len(cut_points) <= bins:
            cut_points.sort()
        elif equi_method == "equif":
            pct = np.arange(1, bins) / bins
            cut_points = var_series.quantile(pct, interpolation="higher").values
        elif equi_method == "equid":
            min_value = min(var_series)
            max_value = max(var_series)
            step = (max_value - min_value) / bins
            cut_points = np.arange(start=min_value, stop=max_value, step=step)[1:]
        else:
            raise ValueError("Invalid equi_method value.")

        # Add the special values that need to be binned separately
        if bool(self.spec_values) and len(self.spec_values) > 0:
            self.spec_values.sort()
            spec_values_upper = [x + 0.1**spec_digit for x in self.spec_values]
            cut_points = np.append(cut_points, self.spec_values + spec_values_upper)
            cut_points.sort()

#         print("NumVarBinning Cut Points: ", cut_points)
        # Return info
        self.equi_method = equi_method

        return cut_points

    def modify_cutpoints(self, df, points):
        """
        Adjust the cut-point sequence.

        1. Cut-point precision:
        (1) If the variable takes integer values, the precision is 1 decimal place
        (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

        2. Filter out cut points that fall outside the value range of the variable, and remove duplicates

        3. For better generalization, the minimum of the cut-point sequence is replaced with -inf and +inf is added as the maximum.

        Parameters
        ----------
        df : pandas.DataFrame
            Data table
        points : array-like
            Cut-point sequence

        Returns
        -------
        cut_points : numpy.array
            Adjusted cut-point sequence
        """
        var_series = df[self.var_name]

        if pd.api.types.is_integer_dtype(var_series):
            spec_digit = 1
        else:
            spec_digit = self.spec_digit

        # Filter the cut points within the value range of the variable
        ## Added
        min_value_wo_spec_values = min(var_series[~var_series.isin(self.spec_values)])
        max_value_wo_spec_values = max(var_series[~var_series.isin(self.spec_values)])

        min_value = min(var_series)
        max_value = max(var_series)
#         print("(MIN, MAX): ", min_value, max_value)
        cut_points = list(filter(lambda x: min_value <= x <= max_value, points))
#         print("Modify Init Cut Points: ", cut_points)
        if bool(self.spec_values):
            spec_cut_points = list(filter(lambda x: min_value <= x <= max_value, self.spec_values))
#             print("Modify Spec Cut Points: ", spec_cut_points)

            ## Added
            # Near the float limits (the -1.797e308 missing sentinel) adding the step changes nothing: take the next
            # representable value so that the special value still ends a bin of its own
            spec_values_upper = [
                x + 0.1**spec_digit if x + 0.1**spec_digit != x else np.nextafter(x, np.inf) for x in self.spec_values
            ]

            cut_points.extend(spec_cut_points)

            ## Added
            cut_points.extend(spec_values_upper)

        cut_points = np.array(cut_points).astype("float")
        with np.errstate(over="ignore"):
            rounded_points = cut_points.round(spec_digit)
        # A point the rounding overflows to infinity (the -1.797e308 sentinel) is kept as it is
        cut_points = np.unique(np.where(np.isfinite(rounded_points) | ~np.isfinite(cut_points), rounded_points, cut_points))
        cut_points = np.insert(cut_points, 0, -np.inf)
        cut_points = np.append(cut_points, np.inf)

#         print("Final Cut Points Before Binning: ", cut_points)
        # Filter to the cut points that have samples
        binning_series = pd.cut(df[self.var_name], cut_points, right=False, labels=cut_points[:-1])
        binning_cnts = binning_series.value_counts()
        cut_points = np.array(binning_cnts.index[binning_cnts > 0])
        cut_points.sort()

        # If the first cut point is not in spec_values, set it to -inf
        if bool(self.spec_values):
            if cut_points[0] not in self.spec_values:
                cut_points[0] = -np.inf
        else:
            cut_points[0] = -np.inf

        # Add +inf as the maximum of the variable
        cut_points = np.append(cut_points, np.inf)

        return cut_points

    def apt_binning(self, df, points, modify=True):
        """
        Bin the variable using the specified cut-point sequence.

        Cut-point precision:
        (1) If the variable takes integer values, the precision is 1 decimal place
        (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

        Filter out cut points that fall outside the value range of the variable

        For better generalization, the minimum of the cut-point sequence is replaced with -inf and +inf is added as the maximum.

        Parameters
        ----------
        df : pandas.DataFrame
            Data table
        points : array-like
            Cut-point sequence
        modify : bool, default True
            Whether to adjust the cut-point sequence

        Returns
        -------
        binning_series : pandas.Categorical
            Binned series
        """
        if modify:
            self.cut_points = self.modify_cutpoints(df, points)
        else:
            self.cut_points = sorted(points)
        self.bins = len(self.cut_points) - 1

        binning_series = pd.cut(df[self.var_name], self.cut_points, right=False)
        self.bin_names = binning_series.values.categories
        # self.bin_names = {i: str(c) for i, c in zip(range(self.bins), binning_series.values.categories)}
        # binning_series = binning_series.values.rename_categories(range(self.bins))

        return binning_series

    def equi_binning(self, df, bins=10, equi_method="equif"):
        """
        Compute the cut-point sequence of a numeric variable from the number of bins and the equi-binning method, then bin the variable.

        Parameters
        ----------
        df : pandas.DataFrame
            Data table
        bins : int, default 10
            Number of bins
        equi_method : string, default "equif"
            Binning method, candidate values {"equid": equal-width, "equif": equal-frequency}

        Returns
        -------
        binning_series : pandas.Categorical
            Binned series
        """
        logging.info(f"-------- [{self.var_name} : EquiX Binning] --------")
        logging.info(f"PARAMS: bins={bins}, equi_method={equi_method}")
        cut_points = self.calc_equi_cutpoints(df=df, bins=bins, equi_method=equi_method)
        binning_series = self.apt_binning(df=df, points=cut_points, modify=True)

        return binning_series

    def apply_binning(self, df):
        """
        Bin the variable using the saved cut-point sequence.

        Bin new data using the cut points previously computed by methods such as equi_binning or auto_binning.

        Parameters
        ----------
        df : pandas.DataFrame
            Data table

        Returns
        -------
        binning_series : pandas.Categorical
            Binned series
        """
        if self.cut_points is None:
            raise ValueError("The class attribute self.cut_points is None.")
        binning_series = self.apt_binning(df, self.cut_points, modify=False)

        return binning_series

    def auto_binning(self, df, tgt_name, max_bins=10, min_prop_in_bin=0.05, equi_bins=200, equi_method="equif", init_points = None,  binning_criteria="chi2", chi2_p=0.95):
        """
        Automatic binning based on the chi-square test.

        Using a distribution-distance measure, automatically derive the bin sequence for the given maximum number of bins and minimum number of samples per bin.
        Steps:
        1. Use the initial binning method to complete the initial binning
        2. Based on the distribution-distance measure, compute the sample size and within-bin distribution distance of each initial bin, as well as the gain in distance difference relative to the original bins after merging adjacent bins
        3. Keep merging bins whose sample size does not meet the minimum-samples-per-bin requirement and bins whose merge with an adjacent bin yields a larger distance-difference gain, until the maximum number of bins and the minimum number of samples per bin are satisfied
        4. Generate the binned variable series from the final binning result.

        Parameters
        ----------
        df : pandas.DataFrame
            Data table
        tgt_name : str
            Target variable name
        max_bins : int, default 10
            Maximum number of bins
        min_prop_in_bin : float, default 0.05
            Minimum proportion of samples per bin
        equi_bins : int, default 200
            Number of initial bins
        equi_method : string, default "equif"
            Initial binning method, candidate values {"equid": equal-width, "equif": equal-frequency}
        init_points : array-like, default None
            Initial cut-point sequence; if not None, equi_bins and equi_method are ignored
        binning_criteria : string, default "chi2"
            Binning criterion, candidate values {"chi2": "chi-square value"}
        chi2_p : float, default 0.95
            Probability used to compute the quantile of the chi-square distribution with 1 degree of freedom. When the chi-square value of adjacent bins is smaller than this quantile, the bins are considered not independent and are merged; a larger value therefore makes the independence test stricter

        Returns
        -------
        binning_series : pandas.Categorical
            Binned series
        """
        logging.info(f"\n---------------- [{self.var_name} : Auto binning] ----------------")
        min_cnt_in_bin = np.floor(df.shape[0] * min_prop_in_bin)
        logging.info(f"PARAMS: max_bins={max_bins}, min_cnt_in_bin={min_cnt_in_bin}, criteria={binning_criteria}")

        # Initial binning
        if init_points is None:
            binning_series = self.equi_binning(df, bins=equi_bins, equi_method=equi_method)
        else:
            binning_series = self.apt_binning(df=df, points=init_points, modify=True)
        cut_points = self.cut_points

        # Pivot table of the initial bins
        df_tmp = df[[self.var_name, tgt_name]].copy()
        df_tmp[self.var_name] = binning_series
        df_pvt = cre_pvt(df=df_tmp, var_name=self.var_name, tgt_name=tgt_name)
        df_pvt.index = cut_points[:-1]

        # List the special values separately
        if bool(self.spec_values):
            df_pvt_spec = df_pvt.loc[[x in self.spec_values for x in df_pvt.index], ].copy()
            df_pvt = df_pvt.loc[[x not in self.spec_values for x in df_pvt.index], ].copy()
            max_bins = max_bins - df_pvt_spec.shape[0]

        # Automatic binning
        if not df_pvt.empty:
            logging.info(f"-------- [{self.var_name} : Auto binning] --------")
            df_pvt = chi2_auto_binning(df_pvt=df_pvt, max_bins=max_bins, min_cnt_in_bin=min_cnt_in_bin, p=chi2_p)

            # Recompute the bins
            cut_points = df_pvt.index

            # Add the special values that need to be binned separately
            if bool(self.spec_values) and len(self.spec_values) > 0:
#                 spec_values_upper = [x + 0.1**spec_digit for x in self.spec_values]
                cut_points = np.append(cut_points, self.spec_values)
#                 cut_points = np.append(cut_points, self.spec_values_upper)
#                 cut_points.sort()

#             print("Cut Points After Auto Binning: ", cut_points)

        binning_series = self.apt_binning(
            df=df, 
            points=cut_points, 
            modify=True
            )

        return binning_series

calc_equi_cutpoints

calc_equi_cutpoints(df, bins=10, equi_method='equif')

Compute the cut-point sequence of a numeric variable from the number of bins and the equi-binning method.

1. Compute the cut points from the specified number of bins; the binning methods are "equal-width" and "equal-frequency" (1) When the number of bins is not smaller than the number of unique values of the variable, the cut-point sequence is the sequence of unique values (2) When the number of bins is smaller than the number of unique values, either "equal-width" or "equal-frequency" binning can be chosen

2. If values that must be binned separately are specified: (1) Add the values to the cut-point sequence; (2) Add the upper bound of each value = value + 0.1**(precision+1) to the cut-point sequence;

3. Cut-point precision: (1) If the variable takes integer values, the precision is 1 decimal place (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

Parameters:

Name	Type	Description	Default
df	DataFrame	Data table	required
bins	int	Number of bins	10
equi_method	string	Binning method, candidate values {"equid": equal-width, "equif": equal-frequency}	"equif"

Returns:

Name	Type	Description
cut_points	array	Cut-point sequence

Source code in Modeling_Tool/Core/Binning_Tool.py

    def calc_equi_cutpoints(self, df, bins=10, equi_method="equif"):
        """
        Compute the cut-point sequence of a numeric variable from the number of bins and the equi-binning method.

        1. Compute the cut points from the specified number of bins; the binning methods are "equal-width" and "equal-frequency"
        (1) When the number of bins is not smaller than the number of unique values of the variable, the cut-point sequence is the sequence of unique values
        (2) When the number of bins is smaller than the number of unique values, either "equal-width" or "equal-frequency" binning can be chosen

        2. If values that must be binned separately are specified:
        (1) Add the values to the cut-point sequence;
        (2) Add the upper bound of each value = value + 0.1**(precision+1) to the cut-point sequence;

        3. Cut-point precision:
        (1) If the variable takes integer values, the precision is 1 decimal place
        (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

        Parameters
        ----------
        df : pandas.DataFrame
            Data table
        bins : int, default 10
            Number of bins
        equi_method : string, default "equif"
            Binning method, candidate values {"equid": equal-width, "equif": equal-frequency}

        Returns
        -------
        cut_points : numpy.array
            Cut-point sequence
        """
        var_series = df[self.var_name]

        if any(pd.isnull(var_series)):
            raise ValueError(f"{self.var_name}: the variable contains NaN values")

        if pd.api.types.is_integer_dtype(var_series):
            spec_digit = 1
        else:
            spec_digit = self.spec_digit

        # Compute the cut-point values from the specified number of bins
        cut_points = var_series.unique()
        if len(cut_points) <= bins:
            cut_points.sort()
        elif equi_method == "equif":
            pct = np.arange(1, bins) / bins
            cut_points = var_series.quantile(pct, interpolation="higher").values
        elif equi_method == "equid":
            min_value = min(var_series)
            max_value = max(var_series)
            step = (max_value - min_value) / bins
            cut_points = np.arange(start=min_value, stop=max_value, step=step)[1:]
        else:
            raise ValueError("Invalid equi_method value.")

        # Add the special values that need to be binned separately
        if bool(self.spec_values) and len(self.spec_values) > 0:
            self.spec_values.sort()
            spec_values_upper = [x + 0.1**spec_digit for x in self.spec_values]
            cut_points = np.append(cut_points, self.spec_values + spec_values_upper)
            cut_points.sort()

#         print("NumVarBinning Cut Points: ", cut_points)
        # Return info
        self.equi_method = equi_method

        return cut_points

modify_cutpoints

modify_cutpoints(df, points)

Adjust the cut-point sequence.

1. Cut-point precision: (1) If the variable takes integer values, the precision is 1 decimal place (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

2. Filter out cut points that fall outside the value range of the variable, and remove duplicates

3. For better generalization, the minimum of the cut-point sequence is replaced with -inf and +inf is added as the maximum.

Parameters:

Name	Type	Description	Default
df	DataFrame	Data table	required
points	array - like	Cut-point sequence	required

Returns:

Name	Type	Description
cut_points	array	Adjusted cut-point sequence

Source code in Modeling_Tool/Core/Binning_Tool.py

    def modify_cutpoints(self, df, points):
        """
        Adjust the cut-point sequence.

        1. Cut-point precision:
        (1) If the variable takes integer values, the precision is 1 decimal place
        (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

        2. Filter out cut points that fall outside the value range of the variable, and remove duplicates

        3. For better generalization, the minimum of the cut-point sequence is replaced with -inf and +inf is added as the maximum.

        Parameters
        ----------
        df : pandas.DataFrame
            Data table
        points : array-like
            Cut-point sequence

        Returns
        -------
        cut_points : numpy.array
            Adjusted cut-point sequence
        """
        var_series = df[self.var_name]

        if pd.api.types.is_integer_dtype(var_series):
            spec_digit = 1
        else:
            spec_digit = self.spec_digit

        # Filter the cut points within the value range of the variable
        ## Added
        min_value_wo_spec_values = min(var_series[~var_series.isin(self.spec_values)])
        max_value_wo_spec_values = max(var_series[~var_series.isin(self.spec_values)])

        min_value = min(var_series)
        max_value = max(var_series)
#         print("(MIN, MAX): ", min_value, max_value)
        cut_points = list(filter(lambda x: min_value <= x <= max_value, points))
#         print("Modify Init Cut Points: ", cut_points)
        if bool(self.spec_values):
            spec_cut_points = list(filter(lambda x: min_value <= x <= max_value, self.spec_values))
#             print("Modify Spec Cut Points: ", spec_cut_points)

            ## Added
            # Near the float limits (the -1.797e308 missing sentinel) adding the step changes nothing: take the next
            # representable value so that the special value still ends a bin of its own
            spec_values_upper = [
                x + 0.1**spec_digit if x + 0.1**spec_digit != x else np.nextafter(x, np.inf) for x in self.spec_values
            ]

            cut_points.extend(spec_cut_points)

            ## Added
            cut_points.extend(spec_values_upper)

        cut_points = np.array(cut_points).astype("float")
        with np.errstate(over="ignore"):
            rounded_points = cut_points.round(spec_digit)
        # A point the rounding overflows to infinity (the -1.797e308 sentinel) is kept as it is
        cut_points = np.unique(np.where(np.isfinite(rounded_points) | ~np.isfinite(cut_points), rounded_points, cut_points))
        cut_points = np.insert(cut_points, 0, -np.inf)
        cut_points = np.append(cut_points, np.inf)

#         print("Final Cut Points Before Binning: ", cut_points)
        # Filter to the cut points that have samples
        binning_series = pd.cut(df[self.var_name], cut_points, right=False, labels=cut_points[:-1])
        binning_cnts = binning_series.value_counts()
        cut_points = np.array(binning_cnts.index[binning_cnts > 0])
        cut_points.sort()

        # If the first cut point is not in spec_values, set it to -inf
        if bool(self.spec_values):
            if cut_points[0] not in self.spec_values:
                cut_points[0] = -np.inf
        else:
            cut_points[0] = -np.inf

        # Add +inf as the maximum of the variable
        cut_points = np.append(cut_points, np.inf)

        return cut_points

apt_binning

apt_binning(df, points, modify=True)

Bin the variable using the specified cut-point sequence.

Cut-point precision: (1) If the variable takes integer values, the precision is 1 decimal place (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

Filter out cut points that fall outside the value range of the variable

For better generalization, the minimum of the cut-point sequence is replaced with -inf and +inf is added as the maximum.

Parameters:

Name	Type	Description	Default
df	DataFrame	Data table	required
points	array - like	Cut-point sequence	required
modify	bool	Whether to adjust the cut-point sequence	True

Returns:

Name	Type	Description
binning_series	Categorical	Binned series

Source code in Modeling_Tool/Core/Binning_Tool.py

def apt_binning(self, df, points, modify=True):
    """
    Bin the variable using the specified cut-point sequence.

    Cut-point precision:
    (1) If the variable takes integer values, the precision is 1 decimal place
    (2) If the variable takes floating-point values, the precision is the number of decimal places of the input

    Filter out cut points that fall outside the value range of the variable

    For better generalization, the minimum of the cut-point sequence is replaced with -inf and +inf is added as the maximum.

    Parameters
    ----------
    df : pandas.DataFrame
        Data table
    points : array-like
        Cut-point sequence
    modify : bool, default True
        Whether to adjust the cut-point sequence

    Returns
    -------
    binning_series : pandas.Categorical
        Binned series
    """
    if modify:
        self.cut_points = self.modify_cutpoints(df, points)
    else:
        self.cut_points = sorted(points)
    self.bins = len(self.cut_points) - 1

    binning_series = pd.cut(df[self.var_name], self.cut_points, right=False)
    self.bin_names = binning_series.values.categories
    # self.bin_names = {i: str(c) for i, c in zip(range(self.bins), binning_series.values.categories)}
    # binning_series = binning_series.values.rename_categories(range(self.bins))

    return binning_series

equi_binning

equi_binning(df, bins=10, equi_method='equif')

Compute the cut-point sequence of a numeric variable from the number of bins and the equi-binning method, then bin the variable.

Parameters:

Name	Type	Description	Default
df	DataFrame	Data table	required
bins	int	Number of bins	10
equi_method	string	Binning method, candidate values {"equid": equal-width, "equif": equal-frequency}	"equif"

Returns:

Name	Type	Description
binning_series	Categorical	Binned series

Source code in Modeling_Tool/Core/Binning_Tool.py

def equi_binning(self, df, bins=10, equi_method="equif"):
    """
    Compute the cut-point sequence of a numeric variable from the number of bins and the equi-binning method, then bin the variable.

    Parameters
    ----------
    df : pandas.DataFrame
        Data table
    bins : int, default 10
        Number of bins
    equi_method : string, default "equif"
        Binning method, candidate values {"equid": equal-width, "equif": equal-frequency}

    Returns
    -------
    binning_series : pandas.Categorical
        Binned series
    """
    logging.info(f"-------- [{self.var_name} : EquiX Binning] --------")
    logging.info(f"PARAMS: bins={bins}, equi_method={equi_method}")
    cut_points = self.calc_equi_cutpoints(df=df, bins=bins, equi_method=equi_method)
    binning_series = self.apt_binning(df=df, points=cut_points, modify=True)

    return binning_series

apply_binning

apply_binning(df)

Bin the variable using the saved cut-point sequence.

Bin new data using the cut points previously computed by methods such as equi_binning or auto_binning.

Parameters:

Name	Type	Description	Default
df	DataFrame	Data table	required

Returns:

Name	Type	Description
binning_series	Categorical	Binned series

Source code in Modeling_Tool/Core/Binning_Tool.py

def apply_binning(self, df):
    """
    Bin the variable using the saved cut-point sequence.

    Bin new data using the cut points previously computed by methods such as equi_binning or auto_binning.

    Parameters
    ----------
    df : pandas.DataFrame
        Data table

    Returns
    -------
    binning_series : pandas.Categorical
        Binned series
    """
    if self.cut_points is None:
        raise ValueError("The class attribute self.cut_points is None.")
    binning_series = self.apt_binning(df, self.cut_points, modify=False)

    return binning_series

auto_binning

auto_binning(df, tgt_name, max_bins=10, min_prop_in_bin=0.05, equi_bins=200, equi_method='equif', init_points=None, binning_criteria='chi2', chi2_p=0.95)

Automatic binning based on the chi-square test.

Using a distribution-distance measure, automatically derive the bin sequence for the given maximum number of bins and minimum number of samples per bin. Steps: 1. Use the initial binning method to complete the initial binning 2. Based on the distribution-distance measure, compute the sample size and within-bin distribution distance of each initial bin, as well as the gain in distance difference relative to the original bins after merging adjacent bins 3. Keep merging bins whose sample size does not meet the minimum-samples-per-bin requirement and bins whose merge with an adjacent bin yields a larger distance-difference gain, until the maximum number of bins and the minimum number of samples per bin are satisfied 4. Generate the binned variable series from the final binning result.

Parameters:

Name	Type	Description	Default
df	DataFrame	Data table	required
tgt_name	str	Target variable name	required
max_bins	int	Maximum number of bins	10
min_prop_in_bin	float	Minimum proportion of samples per bin	0.05
equi_bins	int	Number of initial bins	200
equi_method	string	Initial binning method, candidate values {"equid": equal-width, "equif": equal-frequency}	"equif"
init_points	array - like	Initial cut-point sequence; if not None, equi_bins and equi_method are ignored	None
binning_criteria	string	Binning criterion, candidate values {"chi2": "chi-square value"}	"chi2"
chi2_p	float	Probability used to compute the quantile of the chi-square distribution with 1 degree of freedom. When the chi-square value of adjacent bins is smaller than this quantile, the bins are considered not independent and are merged; a larger value therefore makes the independence test stricter	0.95

Returns:

Name	Type	Description
binning_series	Categorical	Binned series

Source code in Modeling_Tool/Core/Binning_Tool.py

    def auto_binning(self, df, tgt_name, max_bins=10, min_prop_in_bin=0.05, equi_bins=200, equi_method="equif", init_points = None,  binning_criteria="chi2", chi2_p=0.95):
        """
        Automatic binning based on the chi-square test.

        Using a distribution-distance measure, automatically derive the bin sequence for the given maximum number of bins and minimum number of samples per bin.
        Steps:
        1. Use the initial binning method to complete the initial binning
        2. Based on the distribution-distance measure, compute the sample size and within-bin distribution distance of each initial bin, as well as the gain in distance difference relative to the original bins after merging adjacent bins
        3. Keep merging bins whose sample size does not meet the minimum-samples-per-bin requirement and bins whose merge with an adjacent bin yields a larger distance-difference gain, until the maximum number of bins and the minimum number of samples per bin are satisfied
        4. Generate the binned variable series from the final binning result.

        Parameters
        ----------
        df : pandas.DataFrame
            Data table
        tgt_name : str
            Target variable name
        max_bins : int, default 10
            Maximum number of bins
        min_prop_in_bin : float, default 0.05
            Minimum proportion of samples per bin
        equi_bins : int, default 200
            Number of initial bins
        equi_method : string, default "equif"
            Initial binning method, candidate values {"equid": equal-width, "equif": equal-frequency}
        init_points : array-like, default None
            Initial cut-point sequence; if not None, equi_bins and equi_method are ignored
        binning_criteria : string, default "chi2"
            Binning criterion, candidate values {"chi2": "chi-square value"}
        chi2_p : float, default 0.95
            Probability used to compute the quantile of the chi-square distribution with 1 degree of freedom. When the chi-square value of adjacent bins is smaller than this quantile, the bins are considered not independent and are merged; a larger value therefore makes the independence test stricter

        Returns
        -------
        binning_series : pandas.Categorical
            Binned series
        """
        logging.info(f"\n---------------- [{self.var_name} : Auto binning] ----------------")
        min_cnt_in_bin = np.floor(df.shape[0] * min_prop_in_bin)
        logging.info(f"PARAMS: max_bins={max_bins}, min_cnt_in_bin={min_cnt_in_bin}, criteria={binning_criteria}")

        # Initial binning
        if init_points is None:
            binning_series = self.equi_binning(df, bins=equi_bins, equi_method=equi_method)
        else:
            binning_series = self.apt_binning(df=df, points=init_points, modify=True)
        cut_points = self.cut_points

        # Pivot table of the initial bins
        df_tmp = df[[self.var_name, tgt_name]].copy()
        df_tmp[self.var_name] = binning_series
        df_pvt = cre_pvt(df=df_tmp, var_name=self.var_name, tgt_name=tgt_name)
        df_pvt.index = cut_points[:-1]

        # List the special values separately
        if bool(self.spec_values):
            df_pvt_spec = df_pvt.loc[[x in self.spec_values for x in df_pvt.index], ].copy()
            df_pvt = df_pvt.loc[[x not in self.spec_values for x in df_pvt.index], ].copy()
            max_bins = max_bins - df_pvt_spec.shape[0]

        # Automatic binning
        if not df_pvt.empty:
            logging.info(f"-------- [{self.var_name} : Auto binning] --------")
            df_pvt = chi2_auto_binning(df_pvt=df_pvt, max_bins=max_bins, min_cnt_in_bin=min_cnt_in_bin, p=chi2_p)

            # Recompute the bins
            cut_points = df_pvt.index

            # Add the special values that need to be binned separately
            if bool(self.spec_values) and len(self.spec_values) > 0:
#                 spec_values_upper = [x + 0.1**spec_digit for x in self.spec_values]
                cut_points = np.append(cut_points, self.spec_values)
#                 cut_points = np.append(cut_points, self.spec_values_upper)
#                 cut_points.sort()

#             print("Cut Points After Auto Binning: ", cut_points)

        binning_series = self.apt_binning(
            df=df, 
            points=cut_points, 
            modify=True
            )

        return binning_series

Binning

Unified binning class.

Integrate several binning methods, such as quick binning and chi-square binning, behind a unified interface for binning data. Supports equal-frequency/equal-width binning, decision-tree binning, automatic chi-square binning, and other strategies.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input data table (it is copied; the original data is not modified)	required
column	str	Name of the column to bin	required
tgt_name	str	Target variable name (required for chi-square binning)	None
nbins	int	Number of bins	10
precision	int	Precision of the edge values (number of decimal places)	5
min_bin_prop	float	Minimum proportion of samples per bin	0.05
include_missing	bool	Whether to include missing values	True
equal_freq	bool	True for equal-frequency binning, False for equal-width binning	True
bin_colnames	tuple	Tuple of the column names of the binning results	("_bin_num", "_bin_range")
ascending	bool	Whether the bin order is ascending	True
right	bool	Whether the intervals are closed on the right	True
include_lowest	bool	Whether to include the lowest value	False
tree_binning	bool	Whether to use decision-tree binning	False
chi2_method	bool	Whether to use chi-square binning	False
chi2_p	float	Significance level of the chi-square test	0.95
init_equi_bins	int	Number of initial equal-frequency bins	200
fillna	any	Fill value for missing values	-999999
spec_values	list	List of special values	[]
random_state	int	Random seed	42

Attributes:

Name	Type	Description
result	DataFrame	Binning result data
bin_edges	ndarray	Array of bin edges

Examples:

>>> binner = Binning(data, column='income', tgt_name='default', nbins=10)
>>> binner.run()
>>> result, edges = binner.get_result()

>>> # Use chi-square binning
>>> binner = Binning(data, column='income', tgt_name='default', 
...                  nbins=10, chi2_method=True, chi2_p=0.95)
>>> binner.run()

Source code in Modeling_Tool/Core/Binning_Tool.py

class Binning:
    """
    Unified binning class.

    Integrate several binning methods, such as quick binning and chi-square binning, behind a unified interface for binning data.
    Supports equal-frequency/equal-width binning, decision-tree binning, automatic chi-square binning, and other strategies.

    Parameters
    ----------
    data : pandas.DataFrame
        Input data table (it is copied; the original data is not modified)
    column : str
        Name of the column to bin
    tgt_name : str, optional
        Target variable name (required for chi-square binning)
    nbins : int, default 10
        Number of bins
    precision : int, default 5
        Precision of the edge values (number of decimal places)
    min_bin_prop : float, default 0.05
        Minimum proportion of samples per bin
    include_missing : bool, default True
        Whether to include missing values
    equal_freq : bool, default True
        True for equal-frequency binning, False for equal-width binning
    bin_colnames : tuple, default ("_bin_num", "_bin_range")
        Tuple of the column names of the binning results
    ascending : bool, default True
        Whether the bin order is ascending
    right : bool, default True
        Whether the intervals are closed on the right
    include_lowest : bool, default False
        Whether to include the lowest value
    tree_binning : bool, default False
        Whether to use decision-tree binning
    chi2_method : bool, default False
        Whether to use chi-square binning
    chi2_p : float, default 0.95
        Significance level of the chi-square test
    init_equi_bins : int, default 200
        Number of initial equal-frequency bins
    fillna : any, default -999999
        Fill value for missing values
    spec_values : list, default []
        List of special values
    random_state : int, default 42
        Random seed

    Attributes
    ----------
    result : pandas.DataFrame
        Binning result data
    bin_edges : numpy.ndarray
        Array of bin edges

    Examples
    --------
    >>> binner = Binning(data, column='income', tgt_name='default', nbins=10)
    >>> binner.run()
    >>> result, edges = binner.get_result()

    >>> # Use chi-square binning
    >>> binner = Binning(data, column='income', tgt_name='default', 
    ...                  nbins=10, chi2_method=True, chi2_p=0.95)
    >>> binner.run()
    """

    def __init__(self, data, column, tgt_name=None, nbins=10, precision=5, min_bin_prop=0.05,
                 include_missing=True, equal_freq=True, bin_colnames=("_bin_num", "_bin_range"),
                 ascending=True, right=True, include_lowest=False, tree_binning=False,
                 chi2_method=False, chi2_p=0.95, init_equi_bins=200, fillna=-999999,
                 spec_values=[], random_state=42):
        """
        Initialize the binning object.

        Parameters
        ----------
        data : pandas.DataFrame
            Input data table
        column : str
            Name of the column to bin
        tgt_name : str, optional
            Target variable name
        nbins : int, default 10
            Number of bins
        precision : int, default 5
            Precision of the edge values
        min_bin_prop : float, default 0.05
            Minimum proportion of samples per bin
        include_missing : bool, default True
            Whether to include missing values
        equal_freq : bool, default True
            Whether to use equal-frequency (True) or equal-width (False) binning
        bin_colnames : tuple, default ("_bin_num", "_bin_range")
            Column names of the binning results
        ascending : bool, default True
            Whether the bin order is ascending
        right : bool, default True
            Whether the intervals are closed on the right
        include_lowest : bool, default False
            Whether to include the lowest value
        tree_binning : bool, default False
            Whether to use decision-tree binning
        chi2_method : bool, default False
            Whether to use chi-square binning
        chi2_p : float, default 0.95
            Significance level of the chi-square test
        init_equi_bins : int, default 200
            Number of initial equal-frequency bins
        fillna : any, default -999999
            Fill value for missing values
        spec_values : list, default []
            List of special values
        random_state : int, default 42
            Random seed
        """
        self.data = data.copy()
        self.original_data = data.copy()
        self.column = column
        self.tgt_name = tgt_name
        self.nbins = nbins
        self.precision = precision
        self.min_bin_prop = min_bin_prop
        self.include_missing = include_missing
        self.equal_freq = equal_freq
        self.bin_colnames = bin_colnames
        self.ascending = ascending
        self.right = right
        self.include_lowest = include_lowest
        self.tree_binning = tree_binning
        self.chi2_method = chi2_method
        self.chi2_p = chi2_p
        self.init_equi_bins = init_equi_bins
        self.fillna = fillna
        self.spec_values = spec_values
        self.random_state = random_state
        self.result = None
        self.bin_edges = None

    def run_quick_binning(self):
        """
        Run quick binning.

        Run equal-frequency or equal-width binning according to the configured parameters and add the result to the data.

        Returns
        -------
        self
            Returns itself to allow method chaining
        """
        if isinstance(self.nbins, int):
            self.nbins = int(get_max_nbins(data=self.data, nbins=self.nbins, 
                                            min_bin_prop=self.min_bin_prop))

        if not self.include_missing:
            self.data = self.data.dropna(subset=[self.column])

        labels = None
        binned, self.bin_edges = quick_binning(
            data=self.data, 
            column=self.column, 
            nbins=self.nbins, 
            precision=self.precision, 
            equal_freq=self.equal_freq, 
            labels=labels, 
            right=self.right, 
            include_lowest=self.include_lowest,
            tree_binning=self.tree_binning,
            target=self.tgt_name,
            ascending=self.ascending,
            include_missing=self.include_missing,
            random_state=self.random_state,
            spec_values=self.spec_values
        )

        left_sign = '[' if self.include_lowest else '('
        right_sign = ']' if self.right else ')'
        bin_range_list = get_bin_range(edges=self.bin_edges, precision=self.precision, 
                                         ascending=self.ascending, left_sign=left_sign, 
                                         right_sign=right_sign)

        rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not self.include_missing else [i for i in range(0, len(bin_range_list))]
        binned = binned.cat.rename_categories(rename_catlist)

        bin_num_col = self.bin_colnames[0]
        bin_range_col = self.bin_colnames[1]

        self.data = _materialize_bin_columns(
            self.data, binned, bin_range_list, bin_num_col, bin_range_col
        )

        self.result = self.data
        return self

    def run_chi2_binning(self, init_points=None):
        """
        Run chi-square binning.

        Building on quick binning, further apply the chi-square test to optimize the bins automatically.

        Parameters
        ----------
        init_points : array-like, optional
            Initial bin edges, used as the starting point of chi-square binning

        Returns
        -------
        self
            Returns itself to allow method chaining
        """
        self.data = self.original_data.copy()
        self.data = self.data.reset_index(drop=True)

        df = self.data[[self.column, self.tgt_name]].copy()

        equi_method = "equid"
        if self.equal_freq:
            equi_method = "equif"

        if self.include_missing:
            df[self.column] = df[self.column].fillna(self.fillna)
            spec_values = [self.fillna, *self.spec_values]
        else:
            df = df.dropna()
            self.data = self.data[~pd.isnull(self.data[self.column])]

        nvb = NumVarBinning(var_name=self.column, spec_values=spec_values, 
                             spec_digit=self.precision)  
        binning_series = nvb.auto_binning(
            df=df, 
            tgt_name=self.tgt_name, 
            max_bins=self.nbins, 
            min_prop_in_bin=self.min_bin_prop, 
            equi_method=equi_method,
            equi_bins=self.init_equi_bins, 
            binning_criteria='chi2',
            chi2_p=self.chi2_p,
            init_points=init_points
        )

        bin_edges = cat_2_list(binning_series)

        left_sign = '['
        right_sign = ')'
        if not self.ascending:
            left_sign = '('
            right_sign = ']'
        bin_range_list = get_bin_range(edges=bin_edges, precision=self.precision, 
                                         ascending=self.ascending, left_sign=left_sign, 
                                         right_sign=right_sign)

        binned = binning_series
        rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not self.include_missing else [i for i in range(0, len(bin_range_list))]
        binned = binned.cat.rename_categories(rename_catlist)

        bin_num_col = self.bin_colnames[0]
        bin_range_col = self.bin_colnames[1]

        self.data = _materialize_bin_columns(
            self.data, binned, bin_range_list, bin_num_col, bin_range_col
        )

        self.bin_edges = sorted([np.inf if str(x).lower() == 'inf' else -np.inf if str(x).lower() == '-inf' else x for x in bin_edges])

        self.result = self.data
        return self

    def run(self):
        """
        Run the binning.

        Run either quick binning or chi-square binning, depending on the chi2_method parameter.

        Returns
        -------
        self
            Returns itself to allow method chaining
        """
        if self.chi2_method:
            # Run quick binning first to obtain the initial edges
            self.run_quick_binning()
            init_points = self.bin_edges.copy()

            # Then run chi-square binning
            self.run_chi2_binning(init_points=init_points)
        else:
            self.run_quick_binning()

        return self

    def get_result(self, return_edges=True):
        """
        Get the binning result.

        Parameters
        ----------
        return_edges : bool, default True
            Whether to return the bin edges

        Returns
        -------
        tuple or pandas.DataFrame
            If return_edges is True, return the tuple (result, bin_edges);
            otherwise return only result
        """
        if return_edges:
            return self.result, self.bin_edges
        return self.result

run_quick_binning

run_quick_binning()

Run quick binning.

Run equal-frequency or equal-width binning according to the configured parameters and add the result to the data.

Returns:

Type	Description
self	Returns itself to allow method chaining

Source code in Modeling_Tool/Core/Binning_Tool.py

def run_quick_binning(self):
    """
    Run quick binning.

    Run equal-frequency or equal-width binning according to the configured parameters and add the result to the data.

    Returns
    -------
    self
        Returns itself to allow method chaining
    """
    if isinstance(self.nbins, int):
        self.nbins = int(get_max_nbins(data=self.data, nbins=self.nbins, 
                                        min_bin_prop=self.min_bin_prop))

    if not self.include_missing:
        self.data = self.data.dropna(subset=[self.column])

    labels = None
    binned, self.bin_edges = quick_binning(
        data=self.data, 
        column=self.column, 
        nbins=self.nbins, 
        precision=self.precision, 
        equal_freq=self.equal_freq, 
        labels=labels, 
        right=self.right, 
        include_lowest=self.include_lowest,
        tree_binning=self.tree_binning,
        target=self.tgt_name,
        ascending=self.ascending,
        include_missing=self.include_missing,
        random_state=self.random_state,
        spec_values=self.spec_values
    )

    left_sign = '[' if self.include_lowest else '('
    right_sign = ']' if self.right else ')'
    bin_range_list = get_bin_range(edges=self.bin_edges, precision=self.precision, 
                                     ascending=self.ascending, left_sign=left_sign, 
                                     right_sign=right_sign)

    rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not self.include_missing else [i for i in range(0, len(bin_range_list))]
    binned = binned.cat.rename_categories(rename_catlist)

    bin_num_col = self.bin_colnames[0]
    bin_range_col = self.bin_colnames[1]

    self.data = _materialize_bin_columns(
        self.data, binned, bin_range_list, bin_num_col, bin_range_col
    )

    self.result = self.data
    return self

run_chi2_binning

run_chi2_binning(init_points=None)

Run chi-square binning.

Building on quick binning, further apply the chi-square test to optimize the bins automatically.

Parameters:

Name	Type	Description	Default
init_points	array - like	Initial bin edges, used as the starting point of chi-square binning	None

Returns:

Type	Description
self	Returns itself to allow method chaining

Source code in Modeling_Tool/Core/Binning_Tool.py

def run_chi2_binning(self, init_points=None):
    """
    Run chi-square binning.

    Building on quick binning, further apply the chi-square test to optimize the bins automatically.

    Parameters
    ----------
    init_points : array-like, optional
        Initial bin edges, used as the starting point of chi-square binning

    Returns
    -------
    self
        Returns itself to allow method chaining
    """
    self.data = self.original_data.copy()
    self.data = self.data.reset_index(drop=True)

    df = self.data[[self.column, self.tgt_name]].copy()

    equi_method = "equid"
    if self.equal_freq:
        equi_method = "equif"

    if self.include_missing:
        df[self.column] = df[self.column].fillna(self.fillna)
        spec_values = [self.fillna, *self.spec_values]
    else:
        df = df.dropna()
        self.data = self.data[~pd.isnull(self.data[self.column])]

    nvb = NumVarBinning(var_name=self.column, spec_values=spec_values, 
                         spec_digit=self.precision)  
    binning_series = nvb.auto_binning(
        df=df, 
        tgt_name=self.tgt_name, 
        max_bins=self.nbins, 
        min_prop_in_bin=self.min_bin_prop, 
        equi_method=equi_method,
        equi_bins=self.init_equi_bins, 
        binning_criteria='chi2',
        chi2_p=self.chi2_p,
        init_points=init_points
    )

    bin_edges = cat_2_list(binning_series)

    left_sign = '['
    right_sign = ')'
    if not self.ascending:
        left_sign = '('
        right_sign = ']'
    bin_range_list = get_bin_range(edges=bin_edges, precision=self.precision, 
                                     ascending=self.ascending, left_sign=left_sign, 
                                     right_sign=right_sign)

    binned = binning_series
    rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not self.include_missing else [i for i in range(0, len(bin_range_list))]
    binned = binned.cat.rename_categories(rename_catlist)

    bin_num_col = self.bin_colnames[0]
    bin_range_col = self.bin_colnames[1]

    self.data = _materialize_bin_columns(
        self.data, binned, bin_range_list, bin_num_col, bin_range_col
    )

    self.bin_edges = sorted([np.inf if str(x).lower() == 'inf' else -np.inf if str(x).lower() == '-inf' else x for x in bin_edges])

    self.result = self.data
    return self

run

run()

Run the binning.

Run either quick binning or chi-square binning, depending on the chi2_method parameter.

Returns:

Type	Description
self	Returns itself to allow method chaining

Source code in Modeling_Tool/Core/Binning_Tool.py

def run(self):
    """
    Run the binning.

    Run either quick binning or chi-square binning, depending on the chi2_method parameter.

    Returns
    -------
    self
        Returns itself to allow method chaining
    """
    if self.chi2_method:
        # Run quick binning first to obtain the initial edges
        self.run_quick_binning()
        init_points = self.bin_edges.copy()

        # Then run chi-square binning
        self.run_chi2_binning(init_points=init_points)
    else:
        self.run_quick_binning()

    return self

get_result

get_result(return_edges=True)

Get the binning result.

Parameters:

Name	Type	Description	Default
return_edges	bool	Whether to return the bin edges	True

Returns:

Type	Description
tuple or DataFrame	If return_edges is True, return the tuple (result, bin_edges); otherwise return only result

Source code in Modeling_Tool/Core/Binning_Tool.py

def get_result(self, return_edges=True):
    """
    Get the binning result.

    Parameters
    ----------
    return_edges : bool, default True
        Whether to return the bin edges

    Returns
    -------
    tuple or pandas.DataFrame
        If return_edges is True, return the tuple (result, bin_edges);
        otherwise return only result
    """
    if return_edges:
        return self.result, self.bin_edges
    return self.result

get_max_nbins

get_max_nbins(data, nbins, min_bin_prop=0.05)

Compute the maximum number of bins from a given minimum bin proportion.

From the total number of records and the minimum bin proportion, compute the largest number of bins that still meets the minimum sample-size requirement.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input data table	required
nbins	int	Desired number of bins	required
min_bin_prop	float	Minimum proportion of samples in each bin	0.05

Returns:

Type	Description
int	Maximum feasible number of bins

Examples:

>>> get_max_nbins(data, nbins=10, min_bin_prop=0.05)

Source code in Modeling_Tool/Core/Binning_Tool.py

def get_max_nbins(data, nbins, min_bin_prop = 0.05):
    """
    Compute the maximum number of bins from a given minimum bin proportion.

    From the total number of records and the minimum bin proportion, compute the largest number of bins that still meets the minimum sample-size requirement.

    Parameters
    ----------
    data : pandas.DataFrame
        Input data table
    nbins : int
        Desired number of bins
    min_bin_prop : float, default 0.05
        Minimum proportion of samples in each bin

    Returns
    -------
    int
        Maximum feasible number of bins

    Examples
    --------
    >>> get_max_nbins(data, nbins=10, min_bin_prop=0.05)
    """

    n = data.shape[0]
    if n == 0:
        return nbins   # empty data cannot be binned; return the original value

    min_bin_size = min_bin_prop * n

    if min_bin_size == 0:
            # Here min_bin_prop == 0, which degenerates to at most nbins bins
            return min(nbins, n)

    nbins = min(nbins, max(5, n // min_bin_size))
    return nbins

get_decision_tree_binning_edges

get_decision_tree_binning_edges(feature, target, max_leaf_nodes=5, min_samples_leaf=0.05, random_state=42, missing_ref_value=None, spec_values=[])

Find the optimal bins of a continuous variable using a decision tree.

Use a decision-tree classifier to find the optimal bin edges; suitable for automated binning of continuous variables. The algorithm handles missing values and special values automatically and returns suitable bin edges.

Parameters:

Name	Type	Description	Default
feature	array - like	Continuous feature to bin, a pandas Series or a 1-D array	required
target	array - like	Target variable, a binary label (pandas Series or 1-D array)	required
max_leaf_nodes	int	Maximum number of leaf nodes of the decision tree, i.e. the maximum number of bins	5
min_samples_leaf	float	Minimum sample fraction per leaf node; defaults to 0.05 (5%)	0.05
random_state	int	Random seed to make the results reproducible	42
missing_ref_value	any	Reference value for missing values; this value is treated as missing	None
spec_values	list	List of special values; these values are binned separately	[]

Returns:

Name	Type	Description
bin_edges	list	List of bin edges

Examples:

>>> edges = get_decision_tree_binning_edges(feature, target, max_leaf_nodes=5)

Source code in Modeling_Tool/Core/Binning_Tool.py

def get_decision_tree_binning_edges(feature, target, max_leaf_nodes=5, min_samples_leaf=0.05, random_state=42, missing_ref_value = None, spec_values = []):
    """
    Find the optimal bins of a continuous variable using a decision tree.

    Use a decision-tree classifier to find the optimal bin edges; suitable for automated binning of continuous variables.
    The algorithm handles missing values and special values automatically and returns suitable bin edges.

    Parameters
    ----------
    feature : array-like
        Continuous feature to bin, a pandas Series or a 1-D array
    target : array-like
        Target variable, a binary label (pandas Series or 1-D array)
    max_leaf_nodes : int, default 5
        Maximum number of leaf nodes of the decision tree, i.e. the maximum number of bins
    min_samples_leaf : float, default 0.05
        Minimum sample fraction per leaf node; defaults to 0.05 (5%)
    random_state : int, default 42
        Random seed to make the results reproducible
    missing_ref_value : any, optional
        Reference value for missing values; this value is treated as missing
    spec_values : list, optional
        List of special values; these values are binned separately

    Returns
    -------
    bin_edges : list
        List of bin edges

    Examples
    --------
    >>> edges = get_decision_tree_binning_edges(feature, target, max_leaf_nodes=5)
    """
    from sklearn.tree import DecisionTreeClassifier, export_text
    from sklearn.utils import check_random_state

    if missing_ref_value:
        feature = np.where(feature == missing_ref_value, np.nan, feature)

    # Data preprocessing: remove missing values
    df = pd.DataFrame({
        'feature': feature,
        'target': target
    }).dropna()

    if len(spec_values) > 0:
        df = df[~df["feature"].isin(spec_values)]

    feature_clean = df['feature']
    target_clean = df['target']

    # With at most one distinct value there is nothing to split: return what the tree returns when it makes no
    # split (no thresholds, plus the missing reference), so that the caller sees a list of edges in every case.
    if feature_clean.nunique() <= 1:
        logger.info("Warning: feature variance is 0, cannot bin")
        return [missing_ref_value] if missing_ref_value else []

    # Reshape the feature to a 2D array to fit sklearn
    X = feature_clean.values.reshape(-1, 1)
    y = target_clean.values

    # Create the decision-tree classifier
    tree_model = DecisionTreeClassifier(
        max_leaf_nodes=max_leaf_nodes,
        min_samples_leaf=min_samples_leaf,
        random_state=random_state
    )

    # Fit the decision-tree model
    tree_model.fit(X, y)

    # Extract the bin edges
    threshold = tree_model.tree_.threshold
    feature_min = feature_clean.min()
    feature_max = feature_clean.max()

    # Get the split thresholds of the non-leaf nodes and sort them
    bin_edges = sorted([th for th in threshold if th != -2])

    # Add the minimum and maximum values as edges
#     bin_edges = [feature_min] + bin_edges + [feature_max]

    # Remove possibly duplicated edges
    bin_edges = sorted(list(set(bin_edges)))
    if missing_ref_value:
        bin_edges = sorted(list(set(bin_edges + [missing_ref_value])))

#     print(tree_model.get_params())

    return bin_edges

cre_pvt

cre_pvt(df, var_name, tgt_name)

Create a pivot table with the variable as rows and the target variable as columns.

Build a per-bin statistics pivot table from the specified variable and target variable, containing the number of negative samples, the number of positive samples, the total number of samples, and the target rate of each bin.

Parameters:

Name	Type	Description	Default
df	DataFrame	Data table	required
var_name	str	Variable name (the grouping key)	required
tgt_name	str	Target variable name (binary label, 0 and 1)	required

Returns:

Name	Type	Description
df_pvt	DataFrame	Variable pivot table, containing the columns: - 0: number of negative samples - 1: number of positive samples - n: total number of samples - tr: target rate (proportion of positive samples)

Examples:

>>> df_pvt = cre_pvt(df, var_name='income', tgt_name='default')

Source code in Modeling_Tool/Core/Binning_Tool.py

def cre_pvt(df, var_name, tgt_name):
    """
    Create a pivot table with the variable as rows and the target variable as columns.

    Build a per-bin statistics pivot table from the specified variable and target variable, containing the number of negative samples,
    the number of positive samples, the total number of samples, and the target rate of each bin.

    Parameters
    ----------
    df : pandas.DataFrame
        Data table
    var_name : str
        Variable name (the grouping key)
    tgt_name : str
        Target variable name (binary label, 0 and 1)

    Returns
    -------
    df_pvt : pandas.DataFrame
        Variable pivot table, containing the columns:
        - 0: number of negative samples
        - 1: number of positive samples
        - n: total number of samples
        - tr: target rate (proportion of positive samples)

    Examples
    --------
    >>> df_pvt = cre_pvt(df, var_name='income', tgt_name='default')
    """

    df_pvt = df.groupby(var_name, observed=False)[tgt_name].value_counts().unstack().fillna(0)
    df_pvt["n"] = df_pvt[0] + df_pvt[1]
    df_pvt["tr"] = df_pvt[1] / df_pvt["n"]

    return df_pvt

merge_bins

merge_bins(df_pvt, ilocs)

Merge variable bins based on position indices.

Merge the bins at the specified positions, compute the statistics after merging (sample counts, target rate, etc.), and update the chi-square values and target-rate differences.

Important: the index of df_pvt must not be reset

Parameters:

Name	Type	Description	Default
df_pvt	DataFrame	Variable pivot table generated by the cre_pvt function	required
ilocs	list	List of position indices to merge (e.g. [0,1] merges the first two bins)	required

Returns:

Name	Type	Description
df_pvt_new	DataFrame	Variable pivot table after merging

Examples:

>>> df_pvt = merge_bins(df_pvt, ilocs=[0, 1])

Source code in Modeling_Tool/Core/Binning_Tool.py

def merge_bins(df_pvt, ilocs):
    """
    Merge variable bins based on position indices.

    Merge the bins at the specified positions, compute the statistics after merging (sample counts, target rate, etc.),
    and update the chi-square values and target-rate differences.

    Important: the index of df_pvt must not be reset

    Parameters
    ----------
    df_pvt : pandas.DataFrame
        Variable pivot table generated by the cre_pvt function
    ilocs : list
        List of position indices to merge (e.g. [0,1] merges the first two bins)

    Returns
    -------
    df_pvt_new : pandas.DataFrame
        Variable pivot table after merging

    Examples
    --------
    >>> df_pvt = merge_bins(df_pvt, ilocs=[0, 1])
    """
    ilocs.sort()
    df = df_pvt.copy()

    # Aggregate the field values into the smallest position index
    idxes = df.index
    l_idx = idxes[ilocs[0]]
    df.loc[l_idx, [0, 1, "n"]] = np.apply_over_axes(np.sum, df.loc[idxes[ilocs], [0, 1, "n"]], axes=0)[0]
    df.loc[l_idx, "tr"] = df.loc[l_idx, 1] / df.loc[l_idx, "n"]
    df.drop(index=idxes[ilocs[1:]], inplace=True)

    # Recompute tr_diff and chisq
    idxes = df.index
    if "tr_diff" in df.columns:
        if ilocs[0]+1 < df.shape[0]:
            df.loc[idxes[ilocs[0]], "tr_diff"] = df.loc[idxes[ilocs[0]+1], "tr"] - df.loc[idxes[ilocs[0]], "tr"]
        else:
            df.loc[idxes[ilocs[0]], "tr_diff"] = 0

    if "chisq" in df.columns:
        if ilocs[0]+1 < df.shape[0]:
            df.loc[idxes[ilocs[0]], "chisq"] = chi2_contingency(observed_laplace(df.loc[idxes[[ilocs[0], ilocs[0]+1]], [0, 1]]), correction=False)[0]
        else:
            df.loc[idxes[ilocs[0]], "chisq"] = np.inf

    return df

observed_laplace

observed_laplace(observed, digit=6)

Apply a Laplace correction to the values of a contingency table.

Add a small amount (0.1**digit) to every cell of the contingency table to avoid chi-square test problems caused by zero counts.

Parameters:

Name	Type	Description	Default
observed	array_like	Contingency table (a 2-D array or similar structure)	required
digit	int	Number of decimal places of the small amount added to the observed values; if -1, the integer 1 is added	6

Returns:

Name	Type	Description
obs_laplace	ndarray	Contingency table after the Laplace correction

Examples:

>>> observed = [[10, 20], [30, 40]]
>>> obs_laplace = observed_laplace(observed, digit=6)

Source code in Modeling_Tool/Core/Binning_Tool.py

def observed_laplace(observed, digit=6):
    """
    Apply a Laplace correction to the values of a contingency table.

    Add a small amount (0.1**digit) to every cell of the contingency table to avoid
    chi-square test problems caused by zero counts.

    Parameters
    ----------
    observed : array_like
        Contingency table (a 2-D array or similar structure)
    digit : int, default 6
        Number of decimal places of the small amount added to the observed values; if -1, the integer 1 is added

    Returns
    -------
    obs_laplace : numpy.ndarray
        Contingency table after the Laplace correction

    Examples
    --------
    >>> observed = [[10, 20], [30, 40]]
    >>> obs_laplace = observed_laplace(observed, digit=6)
    """
    obs_laplace = np.asarray(observed) + 0.1**digit

    return obs_laplace

cat_2_list

cat_2_list(bin_series)

Convert a binned Categorical series to a list of edge values.

Extract all interval edges from Categorical data and return the de-duplicated list made up of all left and right edges.

Parameters:

Name	Type	Description	Default
bin_series	Categorical	Binned Categorical series	required

Returns:

Type	Description
list	List containing all interval edge values

Examples:

>>> edges = cat_2_list(binning_series)

Source code in Modeling_Tool/Core/Binning_Tool.py

def cat_2_list(bin_series):
    """
    Convert a binned Categorical series to a list of edge values.

    Extract all interval edges from Categorical data and return the de-duplicated list
    made up of all left and right edges.

    Parameters
    ----------
    bin_series : pandas.Categorical
        Binned Categorical series

    Returns
    -------
    list
        List containing all interval edge values

    Examples
    --------
    >>> edges = cat_2_list(binning_series)
    """

    interval_list = bin_series.cat.categories.tolist()
    edges_res = set()
    for x in interval_list:
        edges_res.add(x.left)
        edges_res.add(x.right)
    edges_res = list(edges_res)

    return edges_res

get_bin_range

get_bin_range(edges, precision=5, ascending=False, left_sign='(', right_sign=']')

Generate a list of interval string descriptions from the bin edges.

Convert a list of bin edges into a list of string descriptions with interval signs; supports custom open/closed interval signs and precision.

Parameters:

Name	Type	Description	Default
edges	array - like	List of bin edge values	required
precision	int	Precision of the edge values (number of decimal places)	5
ascending	bool	Whether to sort in ascending order	False
left_sign	str	Left interval sign; '[' means inclusive, '(' means exclusive	'('
right_sign	str	Right interval sign; ']' means inclusive, ')' means exclusive	']'

Returns:

Type	Description
list	List of interval string descriptions

Examples:

>>> edges = [0, 10, 20, 30]
>>> ranges = get_bin_range(edges, precision=0, ascending=True)
['[0, 10]', '[10, 20]', '[20, 30]']

Source code in Modeling_Tool/Core/Binning_Tool.py

def get_bin_range(edges, precision = 5, ascending = False, left_sign = '(', right_sign = ']'):
    """
    Generate a list of interval string descriptions from the bin edges.

    Convert a list of bin edges into a list of string descriptions with interval signs; supports custom
    open/closed interval signs and precision.

    Parameters
    ----------
    edges : array-like
        List of bin edge values
    precision : int, default 5
        Precision of the edge values (number of decimal places)
    ascending : bool, default False
        Whether to sort in ascending order
    left_sign : str, default '('
        Left interval sign; '[' means inclusive, '(' means exclusive
    right_sign : str, default ']'
        Right interval sign; ']' means inclusive, ')' means exclusive

    Returns
    -------
    list
        List of interval string descriptions

    Examples
    --------
    >>> edges = [0, 10, 20, 30]
    >>> ranges = get_bin_range(edges, precision=0, ascending=True)
    ['[0, 10]', '[10, 20]', '[20, 30]']
    """

    i = 0
    reverse = not ascending
    # Very large edges (such as the float maximum) overflow to inf when rounded to the given precision, as they always did;
    # an edge whose overflow would merge it with another one (the -1.797e308 missing sentinel next to -inf) is kept
    round_edge = _round_edge if _overflow_merges_edges(edges) else round
    with np.errstate(over="ignore"):
        edges = sorted([round_edge(x, precision) for x in edges], reverse = reverse)
    res = []
    while i < len(edges) - 1:
        left = edges[i]
        right = edges[i+1]
        res.append(f"{left_sign}{left}, {right}{right_sign}")
        left = right
        i += 1

    return res

get_bin_range_list

get_bin_range_list(data, col='_bin_range')

Convert a column of bin interval strings to a list of edge values.

Parse the bin interval string column of a DataFrame (such as "[0, 10)", "(10, 20]"), extract all unique edge values, and return them sorted.

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the bin interval column	required
col	str	Name of the bin interval column	"_bin_range"

Returns:

Type	Description
list	De-duplicated, sorted list of edge values

Examples:

>>> unique_range = get_bin_range_list(data, col="bin_range")

Source code in Modeling_Tool/Core/Binning_Tool.py

def get_bin_range_list(data, col = "_bin_range"):
    """
    Convert a column of bin interval strings to a list of edge values.

    Parse the bin interval string column of a DataFrame (such as "[0, 10)", "(10, 20]"),
    extract all unique edge values, and return them sorted.

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the bin interval column
    col : str, default "_bin_range"
        Name of the bin interval column

    Returns
    -------
    list
        De-duplicated, sorted list of edge values

    Examples
    --------
    >>> unique_range = get_bin_range_list(data, col="bin_range")
    """

    left, right = _parse_bin_range_bounds(data, col=col)
    return np.unique(np.concatenate([left, right])).tolist()

chi2_auto_binning

chi2_auto_binning(df_pvt, max_bins, min_cnt_in_bin, p=0.95)

Automatic binning based on the chi-square test.

Use the chi-square test to decide whether adjacent bins should be merged, iterating through the following steps: 1. First handle the head and tail bins whose sample size does not meet the minimum requirement 2. Then handle the middle bins with insufficient sample size 3. Finally, merge adjacent bins that are not independent according to the chi-square test

Parameters:

Name	Type	Description	Default
df_pvt	DataFrame	Variable pivot table generated by the cre_pvt function, containing the columns: 0, 1, n, tr	required
max_bins	int	Maximum number of bins	required
min_cnt_in_bin	int	Minimum number of samples per bin	required
p	float	Probability used to compute the quantile of the chi-square distribution with 1 degree of freedom. When the chi-square value of adjacent bins is smaller than this quantile, the bins are considered not independent and are merged. The larger the value, the stricter the independence test	0.95

Returns:

Name	Type	Description
df_pvt	DataFrame	Variable pivot table after merging

Examples:

>>> df_pvt = chi2_auto_binning(df_pvt, max_bins=10, min_cnt_in_bin=100, p=0.95)

Source code in Modeling_Tool/Core/Binning_Tool.py

def chi2_auto_binning(df_pvt, max_bins, min_cnt_in_bin, p=0.95):
    """
    Automatic binning based on the chi-square test.

    Use the chi-square test to decide whether adjacent bins should be merged, iterating through the following steps:
    1. First handle the head and tail bins whose sample size does not meet the minimum requirement
    2. Then handle the middle bins with insufficient sample size
    3. Finally, merge adjacent bins that are not independent according to the chi-square test

    Parameters
    ----------
    df_pvt : pandas.DataFrame
        Variable pivot table generated by the cre_pvt function, containing the columns: 0, 1, n, tr
    max_bins : int
        Maximum number of bins
    min_cnt_in_bin : int
        Minimum number of samples per bin
    p : float, default 0.95
        Probability used to compute the quantile of the chi-square distribution with 1 degree of freedom. When the chi-square value of adjacent bins is smaller than this quantile,
        the bins are considered not independent and are merged. The larger the value, the stricter the independence test

    Returns
    -------
    df_pvt : pandas.DataFrame
        Variable pivot table after merging

    Examples
    --------
    >>> df_pvt = chi2_auto_binning(df_pvt, max_bins=10, min_cnt_in_bin=100, p=0.95)
    """
    # Check whether the sample count meets the minimum-samples requirement
    if np.sum(df_pvt["n"]) <= min_cnt_in_bin:
        df_pvt = merge_bins(df_pvt=df_pvt, ilocs=list(range(df_pvt.shape[0])))
        logging.info("Merge ilocs: all")
    else:
        # Head and tail bins
        # Check whether the bins at the head and tail ends meet the minimum sample requirement
        csumn_asc = np.cumsum(df_pvt["n"])
        ilocs_head = np.min(np.where(csumn_asc >= min_cnt_in_bin))
        if ilocs_head > 0:
            ori_bins = df_pvt.shape[0]
            df_pvt = merge_bins(df_pvt=df_pvt, ilocs=list(range(ilocs_head+1)))
            logging.info(f"HeadMerge: ilocs={[0, ilocs_head+1]}, bins={ori_bins} -> {df_pvt.shape[0]}")

        csumn_desc = np.cumsum(df_pvt["n"][::-1])
        ilocs_tail = np.min(np.where(csumn_desc >= min_cnt_in_bin))
        if ilocs_tail > 0:
            ori_bins = df_pvt.shape[0]
            df_pvt = merge_bins(df_pvt=df_pvt, ilocs=list(range(ori_bins-(ilocs_tail+1), ori_bins)))
            logging.info(f"TailMerge: ilocs={[ori_bins-(ilocs_tail+1), ori_bins]}, bins={ori_bins} -> {df_pvt.shape[0]}")


        # Automatic binning
        # Compute, for each initial bin, the chi-square value, the distribution divergence after merging with the next bin, and the gain
        chisq = [chi2_contingency(observed_laplace(df_pvt.loc[df_pvt.index[[i, i+1]], [0, 1]]), correction=False)[0] for i in range(df_pvt.shape[0] - 1)]
        chisq.append(np.inf)
        tr_diff = np.diff(df_pvt["tr"])
        tr_diff = np.append(tr_diff, 0)
        df_pvt["chisq"] = chisq
        df_pvt["tr_diff"] = tr_diff

        r = 0
        ori_bins = df_pvt.shape[0]
        while df_pvt.shape[0] > max_bins or any(df_pvt["n"] < min_cnt_in_bin):
            r += 1
            # Handle first the bins whose sample size does not meet the minimum, merging forward or backward
            if any(df_pvt["n"] < min_cnt_in_bin):
                iloc = np.min(np.where(df_pvt["n"] < min_cnt_in_bin))
                # Determine the merge direction of the bin: position and monotonicity
                if iloc == 0:
                    iloc_merge = iloc + 1
                elif iloc == df_pvt.shape[0]-1:
                    iloc_merge = iloc - 1
                else:
                    idxes = df_pvt.index
                    if np.abs(df_pvt.loc[idxes[iloc-1], "tr_diff"]) <= np.abs(df_pvt.loc[idxes[iloc], "tr_diff"]):
                        iloc_merge = iloc - 1
                    else:
                        iloc_merge = iloc + 1
            # Then handle the bins with low chi-square values, merging backward
            else:
                chisq_list = df_pvt["chisq"].to_list()
                iloc = chisq_list.index(np.min(chisq_list))
                iloc_merge = iloc + 1

            # Merge the variable bins
            df_pvt = merge_bins(df_pvt=df_pvt, ilocs=[iloc, iloc_merge])
        logging.info(f"LoopMerge: round={r}, bins={ori_bins} -> {df_pvt.shape[0]}")


        # Chi-square test binning
        r = 0
        ori_bins = df_pvt.shape[0]
        while df_pvt["chisq"].min() <= chi2.ppf(p, 1):
            r += 1
            chisq_list = df_pvt["chisq"].to_list()
            iloc = chisq_list.index(np.min(chisq_list))
            df_pvt = merge_bins(df_pvt=df_pvt, ilocs=[iloc, iloc + 1])
        logging.info(f"Chi2Merge: round={r}, bins={ori_bins} -> {df_pvt.shape[0]}")

    return df_pvt

quick_binning

quick_binning(data, column, labels=None, nbins=10, precision=5, equal_freq=True, right=True, include_lowest=False, min_bin_prop=0.05, ascending=True, include_missing=False, tree_binning=False, target=None, random_state=42, fillna=-999999, spec_values=[])

Quick binning function.

Quickly bin the data with equal-frequency or equal-width binning; several binning strategies are supported: - Equal-frequency binning: split by quantiles so that every bin has a similar number of samples - Equal-width binning: split evenly by value intervals - Decision-tree binning: use a decision tree to find the optimal cut points

Parameters:

Name	Type	Description	Default
data	DataFrame	Input data table	required
column	str	Name of the column to bin	required
labels	array - like	Custom bin labels	None
nbins	int or list / tuple	Number of bins (an integer) or the specified bin edges (a list or tuple)	10
precision	int	Precision of the edge values (number of decimal places)	5
equal_freq	bool	True for equal-frequency binning, False for equal-width binning	True
right	bool	Whether the intervals are closed on the right	True
include_lowest	bool	Whether to include the lowest value	False
min_bin_prop	float	Minimum proportion of samples per bin	0.05
ascending	bool	Whether the bin order is ascending	True
include_missing	bool	Whether to include missing values	False
tree_binning	bool	Whether to use decision-tree binning	False
target	str	Target variable name (required for decision-tree binning)	None
random_state	int	Random seed	42
fillna	any	Fill value for missing values	-999999
spec_values	list	List of special values; each is placed in its own bin	[]

Returns:

Name	Type	Description
binned	Categorical	Binned series
bin_edges	ndarray	Array of bin edges

Examples:

>>> binned, edges = quick_binning(data, 'income', nbins=10, equal_freq=True)

Source code in Modeling_Tool/Core/Binning_Tool.py

def quick_binning(data, column, labels = None, nbins = 10, precision = 5, equal_freq = True, right = True, include_lowest = False, 
                  min_bin_prop = 0.05, ascending = True, include_missing = False, tree_binning = False, target = None, random_state=42, 
                  fillna = -999999, spec_values = []):
    """
    Quick binning function.

    Quickly bin the data with equal-frequency or equal-width binning; several binning strategies are supported:
    - Equal-frequency binning: split by quantiles so that every bin has a similar number of samples
    - Equal-width binning: split evenly by value intervals
    - Decision-tree binning: use a decision tree to find the optimal cut points

    Parameters
    ----------
    data : pandas.DataFrame
        Input data table
    column : str
        Name of the column to bin
    labels : array-like, optional
        Custom bin labels
    nbins : int or list/tuple, default 10
        Number of bins (an integer) or the specified bin edges (a list or tuple)
    precision : int, default 5
        Precision of the edge values (number of decimal places)
    equal_freq : bool, default True
        True for equal-frequency binning, False for equal-width binning
    right : bool, default True
        Whether the intervals are closed on the right
    include_lowest : bool, default False
        Whether to include the lowest value
    min_bin_prop : float, default 0.05
        Minimum proportion of samples per bin
    ascending : bool, default True
        Whether the bin order is ascending
    include_missing : bool, default False
        Whether to include missing values
    tree_binning : bool, default False
        Whether to use decision-tree binning
    target : str, optional
        Target variable name (required for decision-tree binning)
    random_state : int, default 42
        Random seed
    fillna : any, default -999999
        Fill value for missing values
    spec_values : list, default []
        List of special values; each is placed in its own bin

    Returns
    -------
    binned : pandas.Categorical
        Binned series
    bin_edges : numpy.ndarray
        Array of bin edges

    Examples
    --------
    >>> binned, edges = quick_binning(data, 'income', nbins=10, equal_freq=True)
    """

    # values near the float limits overflow when rounded, as they always did
    with np.errstate(over="ignore"):
        binning_series = as_binning_numeric(data[column]).round(precision)

    if include_missing:
        binning_series = binning_series.fillna(fillna)
    else:
        binning_series = binning_series.dropna()

    # Determine Binning Intervals
    if isinstance(nbins, int):

        value_no_spec_value = binning_series[~binning_series.isin(spec_values)]
#         print("Tree Binning No Spec Value: ", value_no_spec_value)

        if len(value_no_spec_value) == 0:
            # Return a default empty binning result
            return pd.Series(), []

        if equal_freq:
            nbins = int(get_max_nbins(data, nbins, min_bin_prop))
            # The missing rows (filled with `fillna`) get a bin of their own, as in the equal-width branch: they must not
            # take part in the quantiles, or they join the lowest bin together with real values
            real_values = value_no_spec_value
            missing_edge = []
            if include_missing:
                is_missing = value_no_spec_value == fillna
                if bool(is_missing.any()):
                    real_values = value_no_spec_value[~is_missing]
                    missing_edge = [fillna]
            breakpoints = (
                np.percentile(real_values, [100 / nbins * i for i in range(1, nbins)]) if len(real_values) else []
            )
            breakpoints = list(breakpoints) + missing_edge + spec_values
        else:
            nbins = int(get_max_nbins(data, nbins, min_bin_prop))
            min_value = binning_series.replace(fillna, np.nan).min() if include_missing else value_no_spec_value.min()
#             print("MIN Value: ", min_value)
            breakpoints = np.linspace(min_value, value_no_spec_value.max(), nbins + 1)[1:-1]
            breakpoints = np.sort(np.unique(list(breakpoints) + [fillna])) if include_missing else breakpoints
            breakpoints = list(breakpoints) + spec_values

        if tree_binning:
            breakpoints = get_decision_tree_binning_edges(
                                binning_series, 
                                data[target],
                                max_leaf_nodes = nbins,
                                min_samples_leaf = min_bin_prop,
                                random_state = random_state,
                                missing_ref_value = fillna if include_missing else None,
                                spec_values = spec_values
                            )
            breakpoints = list(breakpoints)
#         print("Tree Output: ", breakpoints)
        breakpoints = [_round_edge(x, precision) for x in breakpoints]
        fnl_breakpoints = np.sort(np.unique([-np.inf, *breakpoints, np.inf]))
#         print("Final Tree Output: ", fnl_breakpoints)

    if isinstance(nbins, list) or isinstance(nbins, tuple):
        fnl_breakpoints = nbins

    if len(spec_values) > 0:
        fnl_breakpoints = sorted(list(set(list(fnl_breakpoints) + spec_values)))

    # pandas rounds the edges to format interval labels; float-max edges overflow
    # to inf there as they always did, so the numpy notice is not useful
    if labels is None and _overflow_merges_edges(fnl_breakpoints):
        # Edges near the float limits (the -1.797e308 missing sentinel) would collapse into the same interval label and
        # merge their bins: number the bins instead (the callers label them from the exact edges)
        labels = list(range(len(fnl_breakpoints) - 1))
    with np.errstate(over="ignore"):
        binned, bin_edges = pd.cut(
            binning_series, 
            bins = fnl_breakpoints, 
            labels = labels, 
            right = right, 
            include_lowest = include_lowest, 
            retbins = True
        )

    orig_cat = [x for x in binned.cat.categories.tolist()]
    if ascending:
        binned = binned
    else:
        orig_cat.reverse()
        binned = binned.cat.reorder_categories([*orig_cat], ordered=True)

    return binned, bin_edges

chi2_binning

chi2_binning(data, column, nbins=10, precision=5, min_bin_prop=0.05, tgt_name=None, include_missing=True, equal_freq=True, bin_colnames=('_bin_num', '_bin_range'), ascending=True, chi2_p=0.95, init_equi_bins=100, fillna=-999999, spec_values=[], init_points=None)

Binning function based on the chi-square test.

Use the chi-square test to find the optimal bin edges automatically: the test decides whether adjacent bins should be merged, yielding statistically significant bins in the end.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input data table	required
column	str	Name of the column to bin	required
nbins	int	Maximum number of bins	10
precision	int	Precision of the edge values (number of decimal places)	5
min_bin_prop	float	Minimum proportion of samples per bin	0.05
tgt_name	str	Target variable name (binary label, 0 and 1)	None
include_missing	bool	Whether to include missing values	True
equal_freq	bool	True for equal-frequency binning, False for equal-width binning	True
bin_colnames	tuple	Tuple of the column names of the binning results	("_bin_num", "_bin_range")
ascending	bool	Whether the bin order is ascending	True
chi2_p	float	Significance level of the chi-square test	0.95
init_equi_bins	int	Number of initial equal-frequency bins	100
fillna	any	Fill value for missing values	-999999
spec_values	list	List of special values	[]
init_points	array - like	Initial bin edges, used as the starting point of chi-square binning	None

Returns:

Type	Description
tuple	(result, bin_edges) - the binning result DataFrame and the array of bin edges

Examples:

>>> result, edges = chi2_binning(data, column='income', tgt_name='default', nbins=10)

Source code in Modeling_Tool/Core/Binning_Tool.py

def chi2_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, tgt_name = None,
                 include_missing = True, equal_freq = True, bin_colnames = ("_bin_num", "_bin_range"), ascending = True, 
                 chi2_p = 0.95, init_equi_bins = 100, fillna = -999999, spec_values = [], init_points = None):
    """
    Binning function based on the chi-square test.

    Use the chi-square test to find the optimal bin edges automatically: the test decides whether adjacent bins should be merged,
    yielding statistically significant bins in the end.

    Parameters
    ----------
    data : pandas.DataFrame
        Input data table
    column : str
        Name of the column to bin
    nbins : int, default 10
        Maximum number of bins
    precision : int, default 5
        Precision of the edge values (number of decimal places)
    min_bin_prop : float, default 0.05
        Minimum proportion of samples per bin
    tgt_name : str
        Target variable name (binary label, 0 and 1)
    include_missing : bool, default True
        Whether to include missing values
    equal_freq : bool, default True
        True for equal-frequency binning, False for equal-width binning
    bin_colnames : tuple, default ("_bin_num", "_bin_range")
        Tuple of the column names of the binning results
    ascending : bool, default True
        Whether the bin order is ascending
    chi2_p : float, default 0.95
        Significance level of the chi-square test
    init_equi_bins : int, default 100
        Number of initial equal-frequency bins
    fillna : any, default -999999
        Fill value for missing values
    spec_values : list, default []
        List of special values
    init_points : array-like, optional
        Initial bin edges, used as the starting point of chi-square binning

    Returns
    -------
    tuple
        (result, bin_edges) - the binning result DataFrame and the array of bin edges

    Examples
    --------
    >>> result, edges = chi2_binning(data, column='income', tgt_name='default', nbins=10)
    """

    data = data.reset_index(drop = True)

    df = data[[column, tgt_name]].copy()

    equi_method = "equid"
    if equal_freq:
        equi_method = "equif"

    if include_missing:
        df[column] = df[column].fillna(fillna)
        spec_values = [fillna, *spec_values]
    else:
        df = df.dropna()
        data = data[~pd.isnull(data[column])]

    nvb = NumVarBinning(var_name=column, spec_values=spec_values, spec_digit=precision)  
#     print(init_points)
    binning_series = nvb.auto_binning(
        df=df, 
        tgt_name=tgt_name, 
        max_bins=nbins, 
        min_prop_in_bin=min_bin_prop, 
        equi_method=equi_method,
        equi_bins=init_equi_bins, 
        binning_criteria='chi2',
        chi2_p=chi2_p,
        init_points=init_points)

    bin_num_col = bin_colnames[0]
    bin_range_col = bin_colnames[1]

    bin_edges = cat_2_list(binning_series)
    if len(bin_edges) < len(binning_series.cat.categories) + 1 and nvb.cut_points is not None:
        # pandas rounds the interval ends of the categories; near the float limits (the -1.797e308 missing sentinel)
        # that overflows to inf and two edges merge, so take the exact cut points instead
        bin_edges = [float(x) for x in nvb.cut_points]

    left_sign='['
    right_sign=')'
    if not ascending:
        left_sign='('
        right_sign=']'
    bin_range_list = get_bin_range(edges = bin_edges, precision = precision, ascending = ascending, left_sign=left_sign, right_sign=right_sign)

    binned = binning_series
    rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not include_missing else [i for i in range(0, len(bin_range_list))]
    binned = binned.cat.rename_categories(rename_catlist)
    data = _materialize_bin_columns(
        data, binned, bin_range_list, bin_num_col, bin_range_col
    )

    fnl_res = data

    bin_edges = sorted([np.inf if str(x).lower() == 'inf' else -np.inf if str(x).lower() == '-inf' else x for x in bin_edges])

    return fnl_res, bin_edges

run_binning

run_binning(data, column, nbins=10, precision=5, min_bin_prop=0.05, include_missing=True, equal_freq=True, bin_colnames=('bin_num', 'bin_range'), ascending=False, right=True, include_lowest=False, tree_binning=False, target=None, random_state=42, spec_values=[], fillna=-999999)

General-purpose binning function supporting equal-frequency or equal-width binning.

Bin a numeric variable with a range of configuration options and return the binned data together with the bin edges.

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the data	required
column	str	Name of the column to bin	required
nbins	int	Number of bins	10
precision	int	Precision of the edge values (number of decimal places)	5
min_bin_prop	float	Minimum proportion of samples per bin	0.05
include_missing	bool	Whether to include missing values	True
equal_freq	bool	True for equal-frequency binning, False for equal-width binning	True
bin_colnames	tuple	Tuple of the column names of the binning results	("bin_num", "bin_range")
ascending	bool	Whether the bin order is ascending	False
right	bool	Whether the intervals are closed on the right	True
include_lowest	bool	Whether to include the lowest value	False
tree_binning	bool	Whether to use decision-tree binning	False
target	str	Target variable name (required for decision-tree binning)	None
random_state	int	Random seed	42
spec_values	list	List of special values	[]
fillna	scalar	Sentinel value that holds the missing values in their own bin when include_missing=True.	-999999

Returns:

Type	Description
tuple	(data, bin_edges) - the data with the binning columns added and the array of bin edges

Examples:

>>> data, edges = run_binning(data, column='income', nbins=10, equal_freq=True)

Source code in Modeling_Tool/Core/Binning_Tool.py

def run_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, include_missing = True, equal_freq = True, 
                bin_colnames = ("bin_num", "bin_range"), ascending = False, right = True, include_lowest = False, 
                tree_binning = False, target = None, random_state=42, spec_values = [], fillna = -999999):
    """
    General-purpose binning function supporting equal-frequency or equal-width binning.

    Bin a numeric variable with a range of configuration options and return the binned data together with the bin edges.

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the data
    column : str
        Name of the column to bin
    nbins : int, default 10
        Number of bins
    precision : int, default 5
        Precision of the edge values (number of decimal places)
    min_bin_prop : float, default 0.05
        Minimum proportion of samples per bin
    include_missing : bool, default True
        Whether to include missing values
    equal_freq : bool, default True
        True for equal-frequency binning, False for equal-width binning
    bin_colnames : tuple, default ("bin_num", "bin_range")
        Tuple of the column names of the binning results
    ascending : bool, default False
        Whether the bin order is ascending
    right : bool, default True
        Whether the intervals are closed on the right
    include_lowest : bool, default False
        Whether to include the lowest value
    tree_binning : bool, default False
        Whether to use decision-tree binning
    target : str, optional
        Target variable name (required for decision-tree binning)
    random_state : int, default 42
        Random seed
    spec_values : list, default []
        List of special values
    fillna : scalar, default -999999
        Sentinel value that holds the missing values in their own bin when `include_missing=True`.

    Returns
    -------
    tuple
        (data, bin_edges) - the data with the binning columns added and the array of bin edges

    Examples
    --------
    >>> data, edges = run_binning(data, column='income', nbins=10, equal_freq=True)

    """

    # New guard: if the data is empty or the specified column is entirely missing, return a placeholder result directly
    if data.empty or data[column].isnull().all():
        # Return a default result containing a missing-values bin
        bin_num_col, bin_range_col = bin_colnames
        data = data.copy()
        data[bin_num_col] = 0 if include_missing else 1
        data[bin_range_col] = "Missing" if include_missing else "All"
        return data, []

    bin_num_col = bin_colnames[0]
    bin_range_col = bin_colnames[1]

    if isinstance(nbins, int):
        nbins = int(get_max_nbins(data = data, nbins = nbins, min_bin_prop = min_bin_prop))

    if not include_missing:
        data = data.dropna(subset = [column])

    labels = None
    binned, bin_edges = quick_binning(data = data, 
                                      column = column, 
                                      nbins = nbins, 
                                      precision = precision, 
                                      equal_freq = equal_freq, 
                                      labels = labels, 
                                      right = right, 
                                      include_lowest = include_lowest,
                                      tree_binning = tree_binning,
                                      target = target,
                                      ascending = ascending,
                                      include_missing = include_missing,
                                      random_state = random_state,
                                      spec_values = spec_values,
                                      fillna = fillna)

    left_sign='[' if include_lowest else '('
    right_sign=']' if right else ')'
    bin_range_list = get_bin_range(edges = bin_edges, precision = precision, ascending = ascending, left_sign=left_sign, right_sign=right_sign)

    rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not include_missing else [i for i in range(0, len(bin_range_list))]
    binned = binned.cat.rename_categories(rename_catlist)
    data = _materialize_bin_columns(
        data, binned, bin_range_list, bin_num_col, bin_range_col
    )

    return data, bin_edges

super_binning

super_binning(data, score, dep, nbins=10, precision=5, min_bin_prop=0.05, include_missing=True, equal_freq=True, chi2_method=False, chi2_p=0.95, init_equi_bins=2000, fillna=-999999, spec_values=[], tree_binning=False, random_state=42, return_edges=False, ascending=True, bin_colnames=('_bin_num', '_bin_range'))

Super binning function that combines several binning strategies.

Provide a unified binning interface with two modes, basic binning and chi-square binning, and automatically choose a suitable binning strategy from the parameter configuration.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input data table	required
score	str	Name of the score/numeric column to bin	required
dep	str	Target variable name (binary label, 0 and 1)	required
nbins	int	Maximum number of bins	10
precision	int	Precision of the edge values (number of decimal places)	5
min_bin_prop	float	Minimum proportion of samples per bin	0.05
include_missing	bool	Whether to include missing values	True
equal_freq	bool	True for equal-frequency binning, False for equal-width binning	True
chi2_method	bool	Whether to refine the bins with chi-square binning	False
chi2_p	float	Significance level of the chi-square test	0.95
init_equi_bins	int	Number of initial equal-frequency bins (before chi-square binning)	2000
fillna	any	Fill value for missing values	-999999
spec_values	list	List of special values	[]
tree_binning	bool	Whether to use decision-tree binning	False
random_state	int	Random seed	42
return_edges	bool	Whether to return the bin edges	False
ascending	bool	Whether the bin order is ascending	True
bin_colnames	tuple	Tuple of the column names of the binning results	("_bin_num", "_bin_range")

Returns:

Type	Description
DataFrame or tuple	If return_edges is False, return the binning result data If return_edges is True, return the tuple (result, edges)

Examples:

>>> # Basic binning
>>> result = super_binning(data, score='income', dep='default', nbins=10)

>>> # Chi-square binning
>>> result, edges = super_binning(data, score='income', dep='default', 
...                                nbins=10, chi2_method=True, return_edges=True)

Source code in Modeling_Tool/Core/Binning_Tool.py

def super_binning(data, score, dep, nbins = 10, precision = 5, min_bin_prop = 0.05, include_missing = True, 
                  equal_freq = True, chi2_method = False, chi2_p = 0.95, init_equi_bins = 2000, fillna = -999999, 
                  spec_values = [], tree_binning = False, random_state=42, return_edges = False, ascending = True,
                  bin_colnames = ("_bin_num", "_bin_range")):
    """
    Super binning function that combines several binning strategies.

    Provide a unified binning interface with two modes, basic binning and chi-square binning,
    and automatically choose a suitable binning strategy from the parameter configuration.

    Parameters
    ----------
    data : pandas.DataFrame
        Input data table
    score : str
        Name of the score/numeric column to bin
    dep : str
        Target variable name (binary label, 0 and 1)
    nbins : int, default 10
        Maximum number of bins
    precision : int, default 5
        Precision of the edge values (number of decimal places)
    min_bin_prop : float, default 0.05
        Minimum proportion of samples per bin
    include_missing : bool, default True
        Whether to include missing values
    equal_freq : bool, default True
        True for equal-frequency binning, False for equal-width binning
    chi2_method : bool, default False
        Whether to refine the bins with chi-square binning
    chi2_p : float, default 0.95
        Significance level of the chi-square test
    init_equi_bins : int, default 2000
        Number of initial equal-frequency bins (before chi-square binning)
    fillna : any, default -999999
        Fill value for missing values
    spec_values : list, default []
        List of special values
    tree_binning : bool, default False
        Whether to use decision-tree binning
    random_state : int, default 42
        Random seed
    return_edges : bool, default False
        Whether to return the bin edges
    ascending : bool, default True
        Whether the bin order is ascending
    bin_colnames : tuple, default ("_bin_num", "_bin_range")
        Tuple of the column names of the binning results

    Returns
    -------
    pandas.DataFrame or tuple
        If return_edges is False, return the binning result data
        If return_edges is True, return the tuple (result, edges)

    Examples
    --------
    >>> # Basic binning
    >>> result = super_binning(data, score='income', dep='default', nbins=10)

    >>> # Chi-square binning
    >>> result, edges = super_binning(data, score='income', dep='default', 
    ...                                nbins=10, chi2_method=True, return_edges=True)
    """

    res, output_edges = run_binning(data = data, 
                                    column = score, 
                                    nbins = nbins, 
                                    precision = precision, 
                                    min_bin_prop = min_bin_prop,
                                    include_missing = include_missing, 
                                    equal_freq = equal_freq,
                                    bin_colnames = bin_colnames,
                                    ascending = ascending,
                                    tree_binning = tree_binning,
                                    target = dep,
                                    random_state = random_state,
                                    spec_values = spec_values,
                                    fillna = fillna)

#     print("First Layer Edges: ", output_edges)

    if chi2_method:
        """ Chi2 Binning. """

#         chi2_edges = [x for x in output_edges if x not in spec_values + [-np.inf, np.inf]]
        chi2_edges = [x for x in output_edges if x not in [-np.inf, np.inf]]

#         print("Special Value: ", spec_values)
#         print("Second Layer Inputs: ", chi2_edges)
        res, output_edges = chi2_binning(data = data, 
                                         column = score, 
                                         tgt_name = dep,
                                         nbins = nbins, 
                                         precision = precision, 
                                         min_bin_prop = min_bin_prop, 
                                         include_missing = include_missing, 
                                         equal_freq = equal_freq, 
                                         bin_colnames = bin_colnames, 
                                         ascending = ascending, 
                                         chi2_p = chi2_p, 
                                         init_equi_bins = init_equi_bins, 
                                         fillna = fillna, 
                                         spec_values = spec_values,
                                         init_points = chi2_edges)


    if return_edges:
        return res, output_edges

    return res

Model Registry: Model_Registry_Tool

save_model, load_model, and load_model_metadata are also importable from Modeling_Tool.

Model_Registry_Tool

Model artifact persistence helpers with metadata support.

This module provides the public save_model / load_model helpers used by Modeling_Tool. It stays backward compatible with legacy files that contain only a raw joblib object while adding an SMF artifact envelope for model version and deployment metadata.

make_model_artifact

make_model_artifact(model, metadata: Optional[Dict[str, Any]] = None, feature_cols=None, woe_mapping_path: Optional[str] = None, train_window: Optional[Dict[str, Any]] = None, metrics: Optional[Dict[str, Any]] = None, model_name: Optional[str] = None, model_version: Optional[str] = None)

Create an in-memory SMF model artifact without writing it to disk.

Parameters:

Name	Type	Description	Default
model	object	Model object to wrap. It is referenced, not copied.	required
metadata	dict	Additional or overriding metadata fields. They are applied last, so a key given here replaces the generated value of the same name (including model_name and model_version passed as arguments).	None
feature_cols	list - like	Training feature list. A single string is stored as a one-element list and any other iterable is stored as a list.	None
woe_mapping_path	str	Path to the WOE mapping table used by the model.	None
train_window	dict	Training / validation / OOT sample window metadata.	None
metrics	dict	Evaluation metrics such as AUC / KS by dataset.	None
model_name	str	Business model name.	None
model_version	str	Business model version.	None

Returns:

Type

Description

dict

The artifact envelope with the keys "__smf_model_artifact__" (True), "artifact_version" ("1.0"), "model" (the object passed in) and "metadata". The "metadata" dict holds smf_version, artifact_version, model_name, model_version, model_class, model_module, feature_cols, woe_mapping_path, train_window, metrics, created_at (UTC ISO timestamp), python_version and platform, updated with the entries of metadata.

Source code in Modeling_Tool/Core/Model_Registry_Tool.py

def make_model_artifact(
    model,
    metadata: Optional[Dict[str, Any]] = None,
    feature_cols=None,
    woe_mapping_path: Optional[str] = None,
    train_window: Optional[Dict[str, Any]] = None,
    metrics: Optional[Dict[str, Any]] = None,
    model_name: Optional[str] = None,
    model_version: Optional[str] = None,
):
    """Create an in-memory SMF model artifact without writing it to disk.

    Parameters
    ----------
    model : object
        Model object to wrap. It is referenced, not copied.
    metadata : dict, optional
        Additional or overriding metadata fields. They are applied last, so a key given here replaces the generated
        value of the same name (including ``model_name`` and ``model_version`` passed as arguments).
    feature_cols : list-like, optional
        Training feature list. A single string is stored as a one-element list and any other iterable is stored as a
        list.
    woe_mapping_path : str, optional
        Path to the WOE mapping table used by the model.
    train_window : dict, optional
        Training / validation / OOT sample window metadata.
    metrics : dict, optional
        Evaluation metrics such as AUC / KS by dataset.
    model_name : str, optional
        Business model name.
    model_version : str, optional
        Business model version.

    Returns
    -------
    dict
        The artifact envelope with the keys ``"__smf_model_artifact__"`` (``True``), ``"artifact_version"``
        (``"1.0"``), ``"model"`` (the object passed in) and ``"metadata"``. The ``"metadata"`` dict holds
        ``smf_version``, ``artifact_version``, ``model_name``, ``model_version``, ``model_class``, ``model_module``,
        ``feature_cols``, ``woe_mapping_path``, ``train_window``, ``metrics``, ``created_at`` (UTC ISO timestamp),
        ``python_version`` and ``platform``, updated with the entries of ``metadata``.
    """
    artifact_metadata = _build_model_metadata(
        model=model,
        metadata=metadata,
        feature_cols=feature_cols,
        woe_mapping_path=woe_mapping_path,
        train_window=train_window,
        metrics=metrics,
        model_name=model_name,
        model_version=model_version,
    )
    return {
        _ARTIFACT_MARKER: True,
        "artifact_version": _ARTIFACT_VERSION,
        "model": model,
        "metadata": artifact_metadata,
    }

save_model

save_model(model, filename, metadata: Optional[Dict[str, Any]] = None, feature_cols=None, woe_mapping_path: Optional[str] = None, train_window: Optional[Dict[str, Any]] = None, metrics: Optional[Dict[str, Any]] = None, model_name: Optional[str] = None, model_version: Optional[str] = None, include_metadata: bool = True)

Save a model, optionally wrapped with standard SMF metadata.

Parameters:

Name	Type	Description	Default
model	object	Model object to persist.	required
filename	str or path - like	Destination path.	required
metadata	dict	Additional or overriding metadata fields.	None
feature_cols	list - like	Training feature list.	None
woe_mapping_path	str	Path to the WOE mapping table used by the model.	None
train_window	dict	Training / validation / OOT sample window metadata.	None
metrics	dict	Evaluation metrics such as AUC / KS by dataset.	None
model_name	str	Business model name.	None
model_version	str	Business model version.	None
include_metadata	bool	If True, save an SMF artifact envelope. If False, save the raw model object exactly like the legacy helper.	True

Returns:

Type	Description
int	0 on success.

Source code in Modeling_Tool/Core/Model_Registry_Tool.py

def save_model(
    model,
    filename,
    metadata: Optional[Dict[str, Any]] = None,
    feature_cols=None,
    woe_mapping_path: Optional[str] = None,
    train_window: Optional[Dict[str, Any]] = None,
    metrics: Optional[Dict[str, Any]] = None,
    model_name: Optional[str] = None,
    model_version: Optional[str] = None,
    include_metadata: bool = True,
):
    """Save a model, optionally wrapped with standard SMF metadata.

    Parameters
    ----------
    model : object
        Model object to persist.
    filename : str or path-like
        Destination path.
    metadata : dict, optional
        Additional or overriding metadata fields.
    feature_cols : list-like, optional
        Training feature list.
    woe_mapping_path : str, optional
        Path to the WOE mapping table used by the model.
    train_window : dict, optional
        Training / validation / OOT sample window metadata.
    metrics : dict, optional
        Evaluation metrics such as AUC / KS by dataset.
    model_name : str, optional
        Business model name.
    model_version : str, optional
        Business model version.
    include_metadata : bool, default True
        If True, save an SMF artifact envelope. If False, save the raw model
        object exactly like the legacy helper.

    Returns
    -------
    int
        0 on success.
    """
    payload = model
    if include_metadata:
        payload = make_model_artifact(
            model=model,
            metadata=metadata,
            feature_cols=feature_cols,
            woe_mapping_path=woe_mapping_path,
            train_window=train_window,
            metrics=metrics,
            model_name=model_name,
            model_version=model_version,
        )
    joblib.dump(payload, filename)
    return 0

load_model

load_model(model_path, return_metadata: bool = False)

Load a legacy model or an SMF model artifact.

By default this keeps backward compatibility and returns only the model object. Set return_metadata=True to receive (model, metadata).

Parameters:

Name	Type	Description	Default
model_path	str or path - like	Path of a file written by save_model (an SMF artifact) or by joblib.dump (a raw model object).	required
return_metadata	bool	If False, return only the model object. If True, return a (model, metadata) tuple.	False

Returns:

Type	Description
object or tuple	The model object, or (model, metadata) when return_metadata=True. metadata is the dict stored in the artifact, and an empty dict {} for a legacy raw-model file.

Notes

The file is read with joblib.load, which unpickles it: load only files from a trusted source.

Source code in Modeling_Tool/Core/Model_Registry_Tool.py

def load_model(model_path, return_metadata: bool = False):
    """Load a legacy model or an SMF model artifact.

    By default this keeps backward compatibility and returns only the model
    object.  Set ``return_metadata=True`` to receive ``(model, metadata)``.

    Parameters
    ----------
    model_path : str or path-like
        Path of a file written by ``save_model`` (an SMF artifact) or by ``joblib.dump`` (a raw model object).
    return_metadata : bool, default False
        If False, return only the model object. If True, return a ``(model, metadata)`` tuple.

    Returns
    -------
    object or tuple
        The model object, or ``(model, metadata)`` when ``return_metadata=True``. ``metadata`` is the dict stored in
        the artifact, and an empty dict ``{}`` for a legacy raw-model file.

    Notes
    -----
    The file is read with ``joblib.load``, which unpickles it: load only files from a trusted source.
    """
    obj = joblib.load(model_path)
    if _is_smf_model_artifact(obj):
        model = obj["model"]
        metadata = obj.get("metadata", {})
        return (model, metadata) if return_metadata else model
    return (obj, {}) if return_metadata else obj

load_model_metadata

load_model_metadata(model_path)

Load only metadata from an SMF model artifact.

Legacy raw-model files return an empty dict.

Parameters:

Name	Type	Description	Default
model_path	str or path - like	Path of a file written by save_model (an SMF artifact) or by joblib.dump (a raw model object).	required

Returns:

Type	Description
dict	The metadata dict stored in the artifact, or an empty dict {} for a legacy raw-model file.

Notes

The whole file is deserialized with joblib.load, model included, so this is not cheaper than load_model, and the same trust requirement applies: load only files from a trusted source.

Source code in Modeling_Tool/Core/Model_Registry_Tool.py

def load_model_metadata(model_path):
    """Load only metadata from an SMF model artifact.

    Legacy raw-model files return an empty dict.

    Parameters
    ----------
    model_path : str or path-like
        Path of a file written by ``save_model`` (an SMF artifact) or by ``joblib.dump`` (a raw model object).

    Returns
    -------
    dict
        The metadata dict stored in the artifact, or an empty dict ``{}`` for a legacy raw-model file.

    Notes
    -----
    The whole file is deserialized with ``joblib.load``, model included, so this is not cheaper than ``load_model``,
    and the same trust requirement applies: load only files from a trusted source.
    """
    obj = joblib.load(model_path)
    if _is_smf_model_artifact(obj):
        return obj.get("metadata", {})
    return {}

ODPS Client: ODPS_Tool

Needs the odps extra, and the ALIBABA_CLOUD_ACCESS_KEY_ID and ALIBABA_CLOUD_ACCESS_KEY_SECRET environment variables when you construct ODPSRunner. See ODPS Data Extraction.

ODPS_Tool

ODPSRunner

Bases: object

Source code in Modeling_Tool/Core/ODPS_Tool.py

class ODPSRunner(object):
    _wide_schema_patch_lock = threading.RLock()
    _wide_schema_patch_ref_count = 0
    _wide_schema_orig_build = None
    _wide_schema_patch_active = False

    """ODPS execution class.

    The connection is configured entirely from environment variables, all four required:
    ``ALIBABA_CLOUD_ACCESS_KEY_ID``, ``ALIBABA_CLOUD_ACCESS_KEY_SECRET``, ``ODPS_PROJECT`` (the project that unqualified
    table names refer to) and ``ODPS_ENDPOINT`` (the MaxCompute endpoint of your region, from Alibaba Cloud's endpoint
    list). A missing or empty variable raises ``KeyError`` naming it; there are no built-in defaults.
    """
    def __init__(self):
        missing = [
            name
            for name in ("ALIBABA_CLOUD_ACCESS_KEY_ID", "ALIBABA_CLOUD_ACCESS_KEY_SECRET", "ODPS_PROJECT", "ODPS_ENDPOINT")
            if not os.environ.get(name)
        ]
        if missing:
            raise KeyError(
                f"ODPSRunner needs the environment variables {missing}; set them before creating the runner "
                "(ODPS_PROJECT is your MaxCompute project and ODPS_ENDPOINT the endpoint of its region)."
            )
        self.o = ODPS(
            os.environ["ALIBABA_CLOUD_ACCESS_KEY_ID"],
            os.environ["ALIBABA_CLOUD_ACCESS_KEY_SECRET"],
            os.environ["ODPS_PROJECT"],
            endpoint=os.environ["ODPS_ENDPOINT"],
        )

        options.retry_times = 6         # number of request retries
        options.pool_maxsize = 200      # maximum connection pool size
        options.connect_timeout = 3600  # connection timeout
        options.read_timeout = 3600     # read timeout


    def run_sql(self, sql, to_df=True, n_process=1, csv_path=None):
        """Run SQL and download the result.

        Parameters
        ----------
        sql : str
            A single SQL statement.
        to_df : bool, default True
            Whether to load the result into memory and return it as a ``pandas.DataFrame``.
            If ``False``, the function returns an **empty DataFrame** but still downloads the data (when ``csv_path`` is specified).
        n_process : int, default 1
            Number of parallel processes for ``executor.open_reader().to_pandas``.
        csv_path : str, default None
            Path of a local CSV file to save the result to. **Independent of ``to_df``**:
                * only ``csv_path`` set: download + write CSV, no data returned (an empty DataFrame is returned)
                * only ``to_df=True`` set: download + return the DataFrame, no CSV written
                * both set: download + return the DataFrame + write CSV
                * neither set: run the SQL only, no download (for DDL/INSERT and similar)

        Returns
        -------
        pandas.DataFrame
            The full data when ``to_df=True``;
            an empty DataFrame (placeholder) when ``to_df=False``.

        Notes
        -----
        - The **execution** phase (execute_sql) runs only once, with no retries.
        - The **download** phase (to_pandas + to_csv) retries up to 6 times, which covers transient network failures.
        - When the SQL returns more than 200 columns, a thread-safe wide-schema patch automatically patches the ODPS Tunnel
          to prevent HTTP 414 (URI too long).

        Examples
        --------
        >>> odps = ODPSRunner()
        >>> df = odps.run_sql("SELECT * FROM dual LIMIT 10")                # DataFrame only
        >>> df = odps.run_sql("SELECT * FROM dual LIMIT 10", csv_path="x.csv")  # DataFrame + CSV
        >>> _  = odps.run_sql("SELECT * FROM dual LIMIT 10", to_df=False,  # CSV only
        ...                   csv_path="x.csv")
        >>> _  = odps.run_sql("CREATE TABLE t AS SELECT 1")                # execute only, no download
        """
        # Prepare the SQL
        sqldesc = sql[:100]+"..." if len(sql)>100 else sql
        logging.info(f"SQL: \n{sqldesc}")

        # Run the SQL (executed once, no retries)
        starttime = datetime.now()
        logging.info(f'  execute_sql: {starttime.strftime("%Y-%m-%d %H:%M:%S")}')
        executor = self.o.execute_sql(sql)

        # Decide whether to download: at least one of to_df=True or a non-empty csv_path is required
        should_download = bool(to_df) or bool(csv_path)
        df = pd.DataFrame()

        if should_download:
            k = 6
            for i in range(k):
                try:
                    logging.info(f'  to_pandas: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
                    reader = executor.open_reader()
                    with self._wide_schema_download_patch(reader):
                        if n_process > 1:
                            df = reader.to_pandas(n_process=n_process)
                        else:
                            df = reader.to_pandas()
                    if bool(csv_path):
                        logging.info(f'  to_csv: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
                        df.to_csv(csv_path, index=False)
                    break
                except Exception as e:
                    logging.error(f'  download failed [{i+1}/{k}]: {e}')
                    if i == k - 1:
                        endtime = datetime.now()
                        duration = round((endtime - starttime).total_seconds(), 1)
                        raise SystemError(
                            f'  break: {endtime.strftime("%Y-%m-%d %H:%M:%S")} duration {duration}\n'
                        )

        # When the caller explicitly does not want a DataFrame, release the reference to save memory
        if not to_df:
            df = pd.DataFrame()

        endtime = datetime.now()
        duration = round((endtime - starttime).total_seconds(), 1)
        logging.info(f'  done: {endtime.strftime("%Y-%m-%d %H:%M:%S")} duration {duration}\n')
        return df

    def download_table(self, table_name, partition=None, n_process=1, csv_path=None):
        """Read table data into a DataFrame.

        Parameters
        ----------
        table_name : str
            Table name
        partition : dict
            Partition, e.g. 'dt=2022-01-01,taino=0'
        n_process : int, default 1
            Number of processes used to convert the query data to a pandas.DataFrame
        csv_path : str
            Path of the CSV file to save the query data to

        Returns
        -------
        df: pandas.DataFrame
            Result data of the SQL query
        """
        logging.info(f"Table: \n{table_name} {partition}")
        starttime = datetime.now()
        logging.info(f'  to_pandas: {starttime.strftime("%Y-%m-%d %H:%M:%S")}')

        t = self.o.get_table(table_name)
        if bool(partition):
            reader = t.open_reader(partition=partition)
        else:
            reader = t.open_reader()
        if n_process > 1:
            df = reader.to_pandas(n_process=10)
        else:
            df = reader.to_pandas()

        if bool(csv_path):
            logging.info(f'  to_csv: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
            df.to_csv(csv_path, index=False)

        endtime = datetime.now()
        duration = round((endtime - starttime).total_seconds(), 3)
        logging.info(f'  done shape={df.shape}: {endtime.strftime("%Y-%m-%d %H:%M:%S")} duration {duration}\n')

        return df

    @classmethod
    @contextmanager
    def _wide_schema_download_patch(cls, reader, col_threshold=200):
        """Prevent HTTP 414 when a query result has many columns.

        ODPS Tunnel encodes column names as URL query params. With 200+ columns
        the URI exceeds the server limit. This context temporarily replaces the
        class-level _build_input_stream to omit the columns param. The patch is
        guarded by a lock and reference count, so concurrent threads do not
        restore the global method while another wide-schema download is active.
        """
        ds = getattr(reader, '_download_session', None)
        if ds is None:
            yield False
            return
        schema = getattr(ds, 'schema', None)
        if schema is None or len(schema.simple_columns) <= col_threshold:
            yield False
            return

        from odps.tunnel.instancetunnel import InstanceDownloadSession

        with cls._wide_schema_patch_lock:
            if not cls._wide_schema_patch_active:
                cls._wide_schema_orig_build = InstanceDownloadSession._build_input_stream

                def _build_no_column_filter(self, start, count, compress=False, columns=None, arrow=False, raw_size=None):
                    with cls._wide_schema_patch_lock:
                        orig_build = cls._wide_schema_orig_build
                    if orig_build is None:
                        raise RuntimeError("ODPS wide-schema download patch lost its original method.")
                    return orig_build(self, start, count, compress=compress, columns=None, arrow=arrow, raw_size=raw_size)

                InstanceDownloadSession._build_input_stream = _build_no_column_filter
                cls._wide_schema_patch_active = True
                logging.info(f'  wide schema ({len(schema.simple_columns)} cols): patched tunnel to omit column filter from URL')
            cls._wide_schema_patch_ref_count += 1

        try:
            yield True
        finally:
            with cls._wide_schema_patch_lock:
                cls._wide_schema_patch_ref_count -= 1
                if cls._wide_schema_patch_ref_count <= 0:
                    if cls._wide_schema_orig_build is not None:
                        InstanceDownloadSession._build_input_stream = cls._wide_schema_orig_build
                    cls._wide_schema_orig_build = None
                    cls._wide_schema_patch_ref_count = 0
                    cls._wide_schema_patch_active = False

    @staticmethod
    def cre_table_schema(df, partition_name=None):
        """Infer an ODPS table schema from the dtypes of a DataFrame.

        Parameters
        ----------
        df : pandas.DataFrame
            Frame whose columns become the table columns.
        partition_name : str or None, default None
            Name of a column of ``df`` to declare as the partition column instead of an ordinary column; None creates
            no partition.

        Returns
        -------
        odps.models.Schema
            One column per DataFrame column (bool -> boolean, integer -> bigint, float32 -> float, other floats ->
            double, datetime -> datetime, string, object and category -> string) and, when ``partition_name`` matches a
            column, one partition.

        Raises
        ------
        TypeError
            If a column has a dtype without an ODPS mapping (for example complex or timedelta).
        """
        table_columns = []
        table_partitions = []
        for col, dtype in df.dtypes.items():
            if pd.api.types.is_complex_dtype(dtype) or pd.api.types.is_timedelta64_dtype(dtype):
                raise TypeError(
                    f"Unsupported dtype for ODPS schema inference: column={col!r}, dtype={dtype!s}."
                )
            if pd.api.types.is_bool_dtype(dtype):
                col_type = "boolean"
            elif pd.api.types.is_integer_dtype(dtype):
                col_type = "bigint"
            elif pd.api.types.is_float_dtype(dtype):
                col_type = "float" if str(dtype).lower() == "float32" else "double"
            elif pd.api.types.is_datetime64_any_dtype(dtype):
                col_type = "datetime"
            elif (
                pd.api.types.is_string_dtype(dtype)
                or pd.api.types.is_object_dtype(dtype)
                or isinstance(dtype, pd.CategoricalDtype)
            ):
                col_type = "string"
            else:
                raise TypeError(
                    f"Unsupported dtype for ODPS schema inference: column={col!r}, dtype={dtype!s}."
                )
            if col == partition_name:
                table_partitions.append(Partition(name=col, type=col_type))
            else:
                table_columns.append(Column(name=col, type=col_type))

        if bool(table_partitions):
            table_schema = Schema(columns=table_columns, partitions=table_partitions)
        else:
            table_schema = Schema(columns=table_columns)

        return table_schema

    @staticmethod
    def _to_odps_records(df):
        """Return Python records with pandas missing values converted to ODPS NULL."""
        object_df = df.astype(object)
        object_df = object_df.where(pd.notna(object_df), None)
        return object_df.values.tolist()

    def _execute_sql_blocking(self, sql):
        """Execute DDL and wait until the remote ODPS instance has completed."""
        execute_sql = getattr(self.o, "execute_sql", None)
        if callable(execute_sql):
            return execute_sql(sql)

        run_sql = getattr(self.o, "run_sql", None)
        if not callable(run_sql):
            raise AttributeError("ODPS client must provide execute_sql() or run_sql().")
        instance = run_sql(sql)
        wait_for_success = getattr(instance, "wait_for_success", None)
        if callable(wait_for_success):
            wait_for_success()
        return instance

    def upload_df(self, df, table_name, table_schema=None, partition=None, atomic=True):
        """Upload a dataset to MaxCompute and create a new table.

        Parameters
        ----------
        df : pandas.DataFrame
            Dataset to upload.
        table_name : str
            Name of the table to create or replace.
        table_schema : odps.models.Schema or None, default None
            Table schema. When None, the schema is inferred from the dtypes of ``df`` (see ``cre_table_schema``) after a
            ``py_inserttime`` column holding the current time has been added to the uploaded data.
        partition : str or None, default None
            Partition spec to write the data to (for example ``"ds='20240101'"``); None writes without a partition.
        atomic : bool, default True
            When ``True`` (recommended, default from 0.4.2), the target table is
            replaced through a temp-table + rename swap so that a failure between
            ``delete_table`` and the completed write never leaves the caller
            with a dropped-and-empty table. When ``False``, the pre-0.4.2
            behaviour is used: the existing table is dropped first and then
            recreated, so any exception during write leaves the target table
            missing. Only set ``atomic=False`` if you deliberately want the
            legacy behaviour, e.g. for tables whose downstream readers expect
            a specific object identity.
        """
        upload_data = df.copy()
        if table_schema is None:
            upload_data.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            table_schema = self.cre_table_schema(df=upload_data, partition_name=None)
        records = self._to_odps_records(upload_data)

        if not atomic:
            # Legacy pre-0.4.2 non-atomic path — retained under an explicit
            # opt-out for callers that need the old identity semantics.
            self.o.delete_table(table_name, if_exists=True)
            t = self.o.create_table(table_name, table_schema)
            if bool(partition):
                with t.open_writer(partition=partition, create_partition=True) as writer:
                    writer.write(records)
            else:
                with t.open_writer() as writer:
                    writer.write(records)
            logger.info(f'<<<< Finished loading data into table {table_name}: shape={df.shape} >>>>')
            return

        # Atomic swap path (default from 0.4.2). If any step before the final
        # swap fails, the original table is untouched.
        ts = datetime.now().strftime('%Y%m%d%H%M%S%f')
        _, target_rename_name = _split_odps_table_name(table_name)
        tmp_name, _ = _make_related_odps_table_name(table_name, f"__tmp_{ts}")
        old_name, old_rename_name = _make_related_odps_table_name(table_name, f"__old_{ts}")

        # Step 1: write data into the temp table. If this fails, drop the
        # partial temp table and re-raise; the target table is untouched.
        self.o.delete_table(tmp_name, if_exists=True)
        tmp_table = self.o.create_table(tmp_name, table_schema)
        try:
            if bool(partition):
                with tmp_table.open_writer(partition=partition, create_partition=True) as writer:
                    writer.write(records)
            else:
                with tmp_table.open_writer() as writer:
                    writer.write(records)
        except Exception:
            try:
                self.o.delete_table(tmp_name, if_exists=True)
            except Exception as cleanup_exc:  # noqa: BLE001
                logger.warning(
                    f"upload_df: failed to drop temp table {tmp_name} after write error: {cleanup_exc!r}"
                )
            raise

        # Step 2: swap. If the original table exists, rename it aside first so
        # that the rename of tmp -> target is guaranteed to succeed. Then drop
        # the old copy (best-effort, warn on failure).
        target_exists = self.o.exist_table(table_name)
        if target_exists:
            try:
                self._execute_sql_blocking(f"ALTER TABLE {table_name} RENAME TO {old_rename_name};")
            except Exception:
                # Rename of the live target failed — do not touch it. Drop the
                # tmp table so we don't leak it, then re-raise.
                try:
                    self.o.delete_table(tmp_name, if_exists=True)
                except Exception as cleanup_exc:  # noqa: BLE001
                    logger.warning(
                        f"upload_df: failed to drop temp table {tmp_name} after rename-target failure: {cleanup_exc!r}"
                    )
                raise

        try:
            self._execute_sql_blocking(f"ALTER TABLE {tmp_name} RENAME TO {target_rename_name};")
        except Exception:
            # tmp -> target rename failed. If we already moved the original,
            # try to restore it so the caller is not left with a missing table.
            if target_exists:
                try:
                    self._execute_sql_blocking(f"ALTER TABLE {old_name} RENAME TO {target_rename_name};")
                    logger.warning(
                        f"upload_df: swap of {tmp_name} -> {table_name} failed; restored original from {old_name}"
                    )
                except Exception as restore_exc:  # noqa: BLE001
                    logger.error(
                        f"upload_df: swap of {tmp_name} -> {table_name} failed AND restore of "
                        f"{old_name} -> {table_name} failed: {restore_exc!r}. "
                        f"Manual recovery required — original data is in {old_name}."
                    )
            try:
                self.o.delete_table(tmp_name, if_exists=True)
            except Exception:  # noqa: BLE001
                pass
            raise

        # Step 3: drop the moved-aside original — best-effort, do not fail the
        # call if this cleanup errors, since the swap already succeeded.
        if target_exists:
            try:
                self.o.delete_table(old_name, if_exists=True)
            except Exception as cleanup_exc:  # noqa: BLE001
                logger.warning(
                    f"upload_df: swap succeeded but failed to drop old copy {old_name}: {cleanup_exc!r}. "
                    f"Safe to delete manually."
                )

        logger.info(f'<<<< Finished loading data into table {table_name}: shape={df.shape} >>>>')

    def _partition_exists(self, table, partition):
        if hasattr(table, "exist_partition"):
            return bool(table.exist_partition(partition))
        try:
            table.get_partition(partition)
            return True
        except Exception:
            return False

    def _rename_partition(self, table_name, source_partition, target_partition):
        src = _format_odps_partition_spec(_parse_odps_partition_spec(source_partition), quoted=True)
        dst = _format_odps_partition_spec(_parse_odps_partition_spec(target_partition), quoted=True)
        self._execute_sql_blocking(
            f"ALTER TABLE {table_name} PARTITION ({src}) RENAME TO PARTITION ({dst});"
        )

    def insert_df(self, df, table_name, overwrite=True, partition=None, atomic=True):
        """Insert a dataset into an existing MaxCompute table.

        Parameters
        ----------
        df : pandas.DataFrame
            Dataset to insert.
        table_name : str
            Name of the existing table.
        overwrite : bool, default True
            Whether to replace the existing data: the target partition is replaced when ``partition`` is given,
            otherwise the whole table is truncated. When False the rows are appended.
        partition : str or None, default None
            Partition spec to write to (for example ``"ds='20240101'"``); the default None writes to the table without
            a partition.
        atomic : bool, default True
            Only used together with ``overwrite=True`` and a ``partition``. When True the data is first written to a
            staging partition that is then swapped in by renaming, so a failed write leaves the previous partition
            untouched; when False the partition is deleted first and then written.

        Notes
        -----
        If the table has a ``py_inserttime`` column that ``df`` lacks, it is filled with the current time.
        """
        t = self.o.get_table(table_name)
        if "py_inserttime" in t.schema and "py_inserttime" not in df.columns:
            df = df.copy()
            df.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        records = self._to_odps_records(df)

        if bool(partition):
            if overwrite and atomic:
                ts = datetime.now().strftime('%Y%m%d%H%M%S%f')
                staging_partition = _make_related_odps_partition_spec(partition, f"__staging_{ts}")
                backup_partition = _make_related_odps_partition_spec(partition, f"__old_{ts}")
                target_exists = self._partition_exists(t, partition)

                for stale in (staging_partition, backup_partition):
                    try:
                        t.delete_partition(stale, if_exists=True)
                    except Exception as cleanup_exc:  # noqa: BLE001
                        logger.warning(
                            f"insert_df: failed to cleanup partition {stale} before atomic write: {cleanup_exc!r}"
                        )

                try:
                    with t.open_writer(partition=staging_partition, create_partition=True) as writer:
                        writer.write(records)
                    if target_exists:
                        self._rename_partition(table_name, partition, backup_partition)
                    self._rename_partition(table_name, staging_partition, partition)
                except Exception:
                    if target_exists:
                        try:
                            if not self._partition_exists(t, partition) and self._partition_exists(t, backup_partition):
                                self._rename_partition(table_name, backup_partition, partition)
                        except Exception as restore_exc:  # noqa: BLE001
                            logger.error(
                                f"insert_df: failed to restore original partition {partition} "
                                f"from {backup_partition}: {restore_exc!r}. Manual recovery may be required."
                            )
                    try:
                        t.delete_partition(staging_partition, if_exists=True)
                    except Exception:  # noqa: BLE001
                        pass
                    raise

                if target_exists:
                    try:
                        t.delete_partition(backup_partition, if_exists=True)
                    except Exception as cleanup_exc:  # noqa: BLE001
                        logger.warning(
                            f"insert_df: atomic swap succeeded but failed to drop backup partition "
                            f"{backup_partition}: {cleanup_exc!r}. Safe to delete manually."
                        )
                logger.info('<<<< insert_df atomic partition write done: shape={0} >>>>'.format(df.shape))
                return

            if overwrite:
                t.delete_partition(partition, if_exists=True)
            with t.open_writer(partition=partition, create_partition=True) as writer:
                writer.write(records)
        else:
            if overwrite:
                t.truncate()
            with t.open_writer() as writer:
                writer.write(records)
        logger.info('<<<< Finished loading data into table: shape={0} >>>>'.format(df.shape))

run_sql

run_sql(sql, to_df=True, n_process=1, csv_path=None)

Run SQL and download the result.

Parameters:

Name	Type	Description	Default
sql	str	A single SQL statement.	required
to_df	bool	Whether to load the result into memory and return it as a pandas.DataFrame. If False, the function returns an empty DataFrame but still downloads the data (when csv_path is specified).	True
n_process	int	Number of parallel processes for executor.open_reader().to_pandas.	1
csv_path	str	Path of a local CSV file to save the result to. Independent of to_df: * only csv_path set: download + write CSV, no data returned (an empty DataFrame is returned) * only to_df=True set: download + return the DataFrame, no CSV written * both set: download + return the DataFrame + write CSV * neither set: run the SQL only, no download (for DDL/INSERT and similar)	None

Returns:

Type	Description
DataFrame	The full data when to_df=True; an empty DataFrame (placeholder) when to_df=False.

Notes

• The execution phase (execute_sql) runs only once, with no retries.
• The download phase (to_pandas + to_csv) retries up to 6 times, which covers transient network failures.
• When the SQL returns more than 200 columns, a thread-safe wide-schema patch automatically patches the ODPS Tunnel to prevent HTTP 414 (URI too long).

Examples:

>>> odps = ODPSRunner()
>>> df = odps.run_sql("SELECT * FROM dual LIMIT 10")                # DataFrame only
>>> df = odps.run_sql("SELECT * FROM dual LIMIT 10", csv_path="x.csv")  # DataFrame + CSV
>>> _  = odps.run_sql("SELECT * FROM dual LIMIT 10", to_df=False,  # CSV only
...                   csv_path="x.csv")
>>> _  = odps.run_sql("CREATE TABLE t AS SELECT 1")                # execute only, no download

Source code in Modeling_Tool/Core/ODPS_Tool.py

def run_sql(self, sql, to_df=True, n_process=1, csv_path=None):
    """Run SQL and download the result.

    Parameters
    ----------
    sql : str
        A single SQL statement.
    to_df : bool, default True
        Whether to load the result into memory and return it as a ``pandas.DataFrame``.
        If ``False``, the function returns an **empty DataFrame** but still downloads the data (when ``csv_path`` is specified).
    n_process : int, default 1
        Number of parallel processes for ``executor.open_reader().to_pandas``.
    csv_path : str, default None
        Path of a local CSV file to save the result to. **Independent of ``to_df``**:
            * only ``csv_path`` set: download + write CSV, no data returned (an empty DataFrame is returned)
            * only ``to_df=True`` set: download + return the DataFrame, no CSV written
            * both set: download + return the DataFrame + write CSV
            * neither set: run the SQL only, no download (for DDL/INSERT and similar)

    Returns
    -------
    pandas.DataFrame
        The full data when ``to_df=True``;
        an empty DataFrame (placeholder) when ``to_df=False``.

    Notes
    -----
    - The **execution** phase (execute_sql) runs only once, with no retries.
    - The **download** phase (to_pandas + to_csv) retries up to 6 times, which covers transient network failures.
    - When the SQL returns more than 200 columns, a thread-safe wide-schema patch automatically patches the ODPS Tunnel
      to prevent HTTP 414 (URI too long).

    Examples
    --------
    >>> odps = ODPSRunner()
    >>> df = odps.run_sql("SELECT * FROM dual LIMIT 10")                # DataFrame only
    >>> df = odps.run_sql("SELECT * FROM dual LIMIT 10", csv_path="x.csv")  # DataFrame + CSV
    >>> _  = odps.run_sql("SELECT * FROM dual LIMIT 10", to_df=False,  # CSV only
    ...                   csv_path="x.csv")
    >>> _  = odps.run_sql("CREATE TABLE t AS SELECT 1")                # execute only, no download
    """
    # Prepare the SQL
    sqldesc = sql[:100]+"..." if len(sql)>100 else sql
    logging.info(f"SQL: \n{sqldesc}")

    # Run the SQL (executed once, no retries)
    starttime = datetime.now()
    logging.info(f'  execute_sql: {starttime.strftime("%Y-%m-%d %H:%M:%S")}')
    executor = self.o.execute_sql(sql)

    # Decide whether to download: at least one of to_df=True or a non-empty csv_path is required
    should_download = bool(to_df) or bool(csv_path)
    df = pd.DataFrame()

    if should_download:
        k = 6
        for i in range(k):
            try:
                logging.info(f'  to_pandas: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
                reader = executor.open_reader()
                with self._wide_schema_download_patch(reader):
                    if n_process > 1:
                        df = reader.to_pandas(n_process=n_process)
                    else:
                        df = reader.to_pandas()
                if bool(csv_path):
                    logging.info(f'  to_csv: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
                    df.to_csv(csv_path, index=False)
                break
            except Exception as e:
                logging.error(f'  download failed [{i+1}/{k}]: {e}')
                if i == k - 1:
                    endtime = datetime.now()
                    duration = round((endtime - starttime).total_seconds(), 1)
                    raise SystemError(
                        f'  break: {endtime.strftime("%Y-%m-%d %H:%M:%S")} duration {duration}\n'
                    )

    # When the caller explicitly does not want a DataFrame, release the reference to save memory
    if not to_df:
        df = pd.DataFrame()

    endtime = datetime.now()
    duration = round((endtime - starttime).total_seconds(), 1)
    logging.info(f'  done: {endtime.strftime("%Y-%m-%d %H:%M:%S")} duration {duration}\n')
    return df

download_table

download_table(table_name, partition=None, n_process=1, csv_path=None)

Read table data into a DataFrame.

Parameters:

Name	Type	Description	Default
table_name	str	Table name	required
partition	dict	Partition, e.g. 'dt=2022-01-01,taino=0'	None
n_process	int	Number of processes used to convert the query data to a pandas.DataFrame	1
csv_path	str	Path of the CSV file to save the query data to	None

Returns:

Name	Type	Description
df	DataFrame	Result data of the SQL query

Source code in Modeling_Tool/Core/ODPS_Tool.py

def download_table(self, table_name, partition=None, n_process=1, csv_path=None):
    """Read table data into a DataFrame.

    Parameters
    ----------
    table_name : str
        Table name
    partition : dict
        Partition, e.g. 'dt=2022-01-01,taino=0'
    n_process : int, default 1
        Number of processes used to convert the query data to a pandas.DataFrame
    csv_path : str
        Path of the CSV file to save the query data to

    Returns
    -------
    df: pandas.DataFrame
        Result data of the SQL query
    """
    logging.info(f"Table: \n{table_name} {partition}")
    starttime = datetime.now()
    logging.info(f'  to_pandas: {starttime.strftime("%Y-%m-%d %H:%M:%S")}')

    t = self.o.get_table(table_name)
    if bool(partition):
        reader = t.open_reader(partition=partition)
    else:
        reader = t.open_reader()
    if n_process > 1:
        df = reader.to_pandas(n_process=10)
    else:
        df = reader.to_pandas()

    if bool(csv_path):
        logging.info(f'  to_csv: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
        df.to_csv(csv_path, index=False)

    endtime = datetime.now()
    duration = round((endtime - starttime).total_seconds(), 3)
    logging.info(f'  done shape={df.shape}: {endtime.strftime("%Y-%m-%d %H:%M:%S")} duration {duration}\n')

    return df

cre_table_schema staticmethod

cre_table_schema(df, partition_name=None)

Infer an ODPS table schema from the dtypes of a DataFrame.

Parameters:

Name	Type	Description	Default
df	DataFrame	Frame whose columns become the table columns.	required
partition_name	str or None	Name of a column of df to declare as the partition column instead of an ordinary column; None creates no partition.	None

Returns:

Type	Description
Schema	One column per DataFrame column (bool -> boolean, integer -> bigint, float32 -> float, other floats -> double, datetime -> datetime, string, object and category -> string) and, when partition_name matches a column, one partition.

Raises:

Type	Description
TypeError	If a column has a dtype without an ODPS mapping (for example complex or timedelta).

Source code in Modeling_Tool/Core/ODPS_Tool.py

@staticmethod
def cre_table_schema(df, partition_name=None):
    """Infer an ODPS table schema from the dtypes of a DataFrame.

    Parameters
    ----------
    df : pandas.DataFrame
        Frame whose columns become the table columns.
    partition_name : str or None, default None
        Name of a column of ``df`` to declare as the partition column instead of an ordinary column; None creates
        no partition.

    Returns
    -------
    odps.models.Schema
        One column per DataFrame column (bool -> boolean, integer -> bigint, float32 -> float, other floats ->
        double, datetime -> datetime, string, object and category -> string) and, when ``partition_name`` matches a
        column, one partition.

    Raises
    ------
    TypeError
        If a column has a dtype without an ODPS mapping (for example complex or timedelta).
    """
    table_columns = []
    table_partitions = []
    for col, dtype in df.dtypes.items():
        if pd.api.types.is_complex_dtype(dtype) or pd.api.types.is_timedelta64_dtype(dtype):
            raise TypeError(
                f"Unsupported dtype for ODPS schema inference: column={col!r}, dtype={dtype!s}."
            )
        if pd.api.types.is_bool_dtype(dtype):
            col_type = "boolean"
        elif pd.api.types.is_integer_dtype(dtype):
            col_type = "bigint"
        elif pd.api.types.is_float_dtype(dtype):
            col_type = "float" if str(dtype).lower() == "float32" else "double"
        elif pd.api.types.is_datetime64_any_dtype(dtype):
            col_type = "datetime"
        elif (
            pd.api.types.is_string_dtype(dtype)
            or pd.api.types.is_object_dtype(dtype)
            or isinstance(dtype, pd.CategoricalDtype)
        ):
            col_type = "string"
        else:
            raise TypeError(
                f"Unsupported dtype for ODPS schema inference: column={col!r}, dtype={dtype!s}."
            )
        if col == partition_name:
            table_partitions.append(Partition(name=col, type=col_type))
        else:
            table_columns.append(Column(name=col, type=col_type))

    if bool(table_partitions):
        table_schema = Schema(columns=table_columns, partitions=table_partitions)
    else:
        table_schema = Schema(columns=table_columns)

    return table_schema

upload_df

upload_df(df, table_name, table_schema=None, partition=None, atomic=True)

Upload a dataset to MaxCompute and create a new table.

Parameters:

Name	Type	Description	Default
df	DataFrame	Dataset to upload.	required
table_name	str	Name of the table to create or replace.	required
table_schema	Schema or None	Table schema. When None, the schema is inferred from the dtypes of df (see cre_table_schema) after a py_inserttime column holding the current time has been added to the uploaded data.	None
partition	str or None	Partition spec to write the data to (for example "ds='20240101'"); None writes without a partition.	None
atomic	bool	When True (recommended, default from 0.4.2), the target table is replaced through a temp-table + rename swap so that a failure between delete_table and the completed write never leaves the caller with a dropped-and-empty table. When False, the pre-0.4.2 behaviour is used: the existing table is dropped first and then recreated, so any exception during write leaves the target table missing. Only set atomic=False if you deliberately want the legacy behaviour, e.g. for tables whose downstream readers expect a specific object identity.	True

Source code in Modeling_Tool/Core/ODPS_Tool.py

def upload_df(self, df, table_name, table_schema=None, partition=None, atomic=True):
    """Upload a dataset to MaxCompute and create a new table.

    Parameters
    ----------
    df : pandas.DataFrame
        Dataset to upload.
    table_name : str
        Name of the table to create or replace.
    table_schema : odps.models.Schema or None, default None
        Table schema. When None, the schema is inferred from the dtypes of ``df`` (see ``cre_table_schema``) after a
        ``py_inserttime`` column holding the current time has been added to the uploaded data.
    partition : str or None, default None
        Partition spec to write the data to (for example ``"ds='20240101'"``); None writes without a partition.
    atomic : bool, default True
        When ``True`` (recommended, default from 0.4.2), the target table is
        replaced through a temp-table + rename swap so that a failure between
        ``delete_table`` and the completed write never leaves the caller
        with a dropped-and-empty table. When ``False``, the pre-0.4.2
        behaviour is used: the existing table is dropped first and then
        recreated, so any exception during write leaves the target table
        missing. Only set ``atomic=False`` if you deliberately want the
        legacy behaviour, e.g. for tables whose downstream readers expect
        a specific object identity.
    """
    upload_data = df.copy()
    if table_schema is None:
        upload_data.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        table_schema = self.cre_table_schema(df=upload_data, partition_name=None)
    records = self._to_odps_records(upload_data)

    if not atomic:
        # Legacy pre-0.4.2 non-atomic path — retained under an explicit
        # opt-out for callers that need the old identity semantics.
        self.o.delete_table(table_name, if_exists=True)
        t = self.o.create_table(table_name, table_schema)
        if bool(partition):
            with t.open_writer(partition=partition, create_partition=True) as writer:
                writer.write(records)
        else:
            with t.open_writer() as writer:
                writer.write(records)
        logger.info(f'<<<< Finished loading data into table {table_name}: shape={df.shape} >>>>')
        return

    # Atomic swap path (default from 0.4.2). If any step before the final
    # swap fails, the original table is untouched.
    ts = datetime.now().strftime('%Y%m%d%H%M%S%f')
    _, target_rename_name = _split_odps_table_name(table_name)
    tmp_name, _ = _make_related_odps_table_name(table_name, f"__tmp_{ts}")
    old_name, old_rename_name = _make_related_odps_table_name(table_name, f"__old_{ts}")

    # Step 1: write data into the temp table. If this fails, drop the
    # partial temp table and re-raise; the target table is untouched.
    self.o.delete_table(tmp_name, if_exists=True)
    tmp_table = self.o.create_table(tmp_name, table_schema)
    try:
        if bool(partition):
            with tmp_table.open_writer(partition=partition, create_partition=True) as writer:
                writer.write(records)
        else:
            with tmp_table.open_writer() as writer:
                writer.write(records)
    except Exception:
        try:
            self.o.delete_table(tmp_name, if_exists=True)
        except Exception as cleanup_exc:  # noqa: BLE001
            logger.warning(
                f"upload_df: failed to drop temp table {tmp_name} after write error: {cleanup_exc!r}"
            )
        raise

    # Step 2: swap. If the original table exists, rename it aside first so
    # that the rename of tmp -> target is guaranteed to succeed. Then drop
    # the old copy (best-effort, warn on failure).
    target_exists = self.o.exist_table(table_name)
    if target_exists:
        try:
            self._execute_sql_blocking(f"ALTER TABLE {table_name} RENAME TO {old_rename_name};")
        except Exception:
            # Rename of the live target failed — do not touch it. Drop the
            # tmp table so we don't leak it, then re-raise.
            try:
                self.o.delete_table(tmp_name, if_exists=True)
            except Exception as cleanup_exc:  # noqa: BLE001
                logger.warning(
                    f"upload_df: failed to drop temp table {tmp_name} after rename-target failure: {cleanup_exc!r}"
                )
            raise

    try:
        self._execute_sql_blocking(f"ALTER TABLE {tmp_name} RENAME TO {target_rename_name};")
    except Exception:
        # tmp -> target rename failed. If we already moved the original,
        # try to restore it so the caller is not left with a missing table.
        if target_exists:
            try:
                self._execute_sql_blocking(f"ALTER TABLE {old_name} RENAME TO {target_rename_name};")
                logger.warning(
                    f"upload_df: swap of {tmp_name} -> {table_name} failed; restored original from {old_name}"
                )
            except Exception as restore_exc:  # noqa: BLE001
                logger.error(
                    f"upload_df: swap of {tmp_name} -> {table_name} failed AND restore of "
                    f"{old_name} -> {table_name} failed: {restore_exc!r}. "
                    f"Manual recovery required — original data is in {old_name}."
                )
        try:
            self.o.delete_table(tmp_name, if_exists=True)
        except Exception:  # noqa: BLE001
            pass
        raise

    # Step 3: drop the moved-aside original — best-effort, do not fail the
    # call if this cleanup errors, since the swap already succeeded.
    if target_exists:
        try:
            self.o.delete_table(old_name, if_exists=True)
        except Exception as cleanup_exc:  # noqa: BLE001
            logger.warning(
                f"upload_df: swap succeeded but failed to drop old copy {old_name}: {cleanup_exc!r}. "
                f"Safe to delete manually."
            )

    logger.info(f'<<<< Finished loading data into table {table_name}: shape={df.shape} >>>>')

insert_df

insert_df(df, table_name, overwrite=True, partition=None, atomic=True)

Insert a dataset into an existing MaxCompute table.

Parameters:

Name	Type	Description	Default
df	DataFrame	Dataset to insert.	required
table_name	str	Name of the existing table.	required
overwrite	bool	Whether to replace the existing data: the target partition is replaced when partition is given, otherwise the whole table is truncated. When False the rows are appended.	True
partition	str or None	Partition spec to write to (for example "ds='20240101'"); the default None writes to the table without a partition.	None
atomic	bool	Only used together with overwrite=True and a partition. When True the data is first written to a staging partition that is then swapped in by renaming, so a failed write leaves the previous partition untouched; when False the partition is deleted first and then written.	True

Notes

If the table has a py_inserttime column that df lacks, it is filled with the current time.

Source code in Modeling_Tool/Core/ODPS_Tool.py

def insert_df(self, df, table_name, overwrite=True, partition=None, atomic=True):
    """Insert a dataset into an existing MaxCompute table.

    Parameters
    ----------
    df : pandas.DataFrame
        Dataset to insert.
    table_name : str
        Name of the existing table.
    overwrite : bool, default True
        Whether to replace the existing data: the target partition is replaced when ``partition`` is given,
        otherwise the whole table is truncated. When False the rows are appended.
    partition : str or None, default None
        Partition spec to write to (for example ``"ds='20240101'"``); the default None writes to the table without
        a partition.
    atomic : bool, default True
        Only used together with ``overwrite=True`` and a ``partition``. When True the data is first written to a
        staging partition that is then swapped in by renaming, so a failed write leaves the previous partition
        untouched; when False the partition is deleted first and then written.

    Notes
    -----
    If the table has a ``py_inserttime`` column that ``df`` lacks, it is filled with the current time.
    """
    t = self.o.get_table(table_name)
    if "py_inserttime" in t.schema and "py_inserttime" not in df.columns:
        df = df.copy()
        df.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    records = self._to_odps_records(df)

    if bool(partition):
        if overwrite and atomic:
            ts = datetime.now().strftime('%Y%m%d%H%M%S%f')
            staging_partition = _make_related_odps_partition_spec(partition, f"__staging_{ts}")
            backup_partition = _make_related_odps_partition_spec(partition, f"__old_{ts}")
            target_exists = self._partition_exists(t, partition)

            for stale in (staging_partition, backup_partition):
                try:
                    t.delete_partition(stale, if_exists=True)
                except Exception as cleanup_exc:  # noqa: BLE001
                    logger.warning(
                        f"insert_df: failed to cleanup partition {stale} before atomic write: {cleanup_exc!r}"
                    )

            try:
                with t.open_writer(partition=staging_partition, create_partition=True) as writer:
                    writer.write(records)
                if target_exists:
                    self._rename_partition(table_name, partition, backup_partition)
                self._rename_partition(table_name, staging_partition, partition)
            except Exception:
                if target_exists:
                    try:
                        if not self._partition_exists(t, partition) and self._partition_exists(t, backup_partition):
                            self._rename_partition(table_name, backup_partition, partition)
                    except Exception as restore_exc:  # noqa: BLE001
                        logger.error(
                            f"insert_df: failed to restore original partition {partition} "
                            f"from {backup_partition}: {restore_exc!r}. Manual recovery may be required."
                        )
                try:
                    t.delete_partition(staging_partition, if_exists=True)
                except Exception:  # noqa: BLE001
                    pass
                raise

            if target_exists:
                try:
                    t.delete_partition(backup_partition, if_exists=True)
                except Exception as cleanup_exc:  # noqa: BLE001
                    logger.warning(
                        f"insert_df: atomic swap succeeded but failed to drop backup partition "
                        f"{backup_partition}: {cleanup_exc!r}. Safe to delete manually."
                    )
            logger.info('<<<< insert_df atomic partition write done: shape={0} >>>>'.format(df.shape))
            return

        if overwrite:
            t.delete_partition(partition, if_exists=True)
        with t.open_writer(partition=partition, create_partition=True) as writer:
            writer.write(records)
    else:
        if overwrite:
            t.truncate()
        with t.open_writer() as writer:
            writer.write(records)
    logger.info('<<<< Finished loading data into table: shape={0} >>>>'.format(df.shape))

ODPS Concurrency: Parallel_ODPS_Manager

Parallel_ODPS_Manager

ParallelODPSConfig dataclass

Settings of a ParallelODPSManager.

Parameters:

Name	Type	Description	Default
unique_key	str or None	Column whose hash splits a pulled query into chunks (ABS(HASH(unique_key)) % n_chunks = chunk_id). It must be visible where {chunk_filter} is placed in the SQL template. With None, pull_split_strategy="auto" uses the ROW_NUMBER strategy.	None
chunk_size	int or None	Rows per chunk. pull derives the number of chunks from the row count (count_query or an automatic COUNT); push splits the data into chunks of this many rows. Cannot be combined with n_chunks.	None
n_chunks	int or None	Number of chunks. pull and push need either this or chunk_size.	None
n_jobs	int	Number of parallel workers; must be positive.	3
backend	('thread', 'process', 'sequential')	Execution backend of the chunk workers. With "process" every worker creates its own ODPSRunner.	"thread"
pull_split_strategy	('auto', 'hash', 'row_number')	How pull splits the query: "hash" needs unique_key, "row_number" first materializes the query in a staging table with a ROW_NUMBER column, and "auto" picks "hash" when unique_key is set and "row_number" otherwise.	"auto"
row_number_order_by	str or None	ORDER BY expression of the ROW_NUMBER window in the staging table; None means ORDER BY 1.	None
row_number_col	str	Name of the helper row-number column of the staging table; it is dropped from the pulled data. Must not be empty.	"__smf_parallel_odps_rn__"
validate_unique_key	bool	In hash mode, run a LIMIT 1 probe of the first chunk before pulling, so that a unique_key that is not visible in the SQL scope fails early with ValueError.	True
chunk_filter_key	str	Name of the placeholder that the SQL template must contain ({chunk_filter} by default). Must not be empty.	"chunk_filter"
tmp_dir	Path or str	Local folder for chunk files; it is created when needed.	Path("data/_chunks")
tmp_table_prefix	str	Prefix of the temporary ODPS tables (the staging table of pull and the chunk tables of push). Must not be empty.	"tmp_parallel_odps"
cleanup_tmp	bool	Drop the temporary ODPS tables when the run ends.	True
keep_tmp_on_error	bool	Keep the temporary tables when the run fails, for debugging. Only relevant when cleanup_tmp is True.	False

Source code in Modeling_Tool/Core/Parallel_ODPS_Manager.py

@dataclass
class ParallelODPSConfig:
    """Settings of a ``ParallelODPSManager``.

    Parameters
    ----------
    unique_key : str or None, default None
        Column whose hash splits a pulled query into chunks (``ABS(HASH(unique_key)) % n_chunks = chunk_id``). It must
        be visible where ``{chunk_filter}`` is placed in the SQL template. With None, ``pull_split_strategy="auto"``
        uses the ROW_NUMBER strategy.
    chunk_size : int or None, default None
        Rows per chunk. ``pull`` derives the number of chunks from the row count (``count_query`` or an automatic
        ``COUNT``); ``push`` splits the data into chunks of this many rows. Cannot be combined with ``n_chunks``.
    n_chunks : int or None, default None
        Number of chunks. ``pull`` and ``push`` need either this or ``chunk_size``.
    n_jobs : int, default 3
        Number of parallel workers; must be positive.
    backend : {"thread", "process", "sequential"}, default "thread"
        Execution backend of the chunk workers. With ``"process"`` every worker creates its own ``ODPSRunner``.
    pull_split_strategy : {"auto", "hash", "row_number"}, default "auto"
        How ``pull`` splits the query: ``"hash"`` needs ``unique_key``, ``"row_number"`` first materializes the query
        in a staging table with a ROW_NUMBER column, and ``"auto"`` picks ``"hash"`` when ``unique_key`` is set and
        ``"row_number"`` otherwise.
    row_number_order_by : str or None, default None
        ``ORDER BY`` expression of the ROW_NUMBER window in the staging table; None means ``ORDER BY 1``.
    row_number_col : str, default "__smf_parallel_odps_rn__"
        Name of the helper row-number column of the staging table; it is dropped from the pulled data. Must not be empty.
    validate_unique_key : bool, default True
        In hash mode, run a ``LIMIT 1`` probe of the first chunk before pulling, so that a ``unique_key`` that is not
        visible in the SQL scope fails early with ``ValueError``.
    chunk_filter_key : str, default "chunk_filter"
        Name of the placeholder that the SQL template must contain (``{chunk_filter}`` by default). Must not be empty.
    tmp_dir : pathlib.Path or str, default Path("data/_chunks")
        Local folder for chunk files; it is created when needed.
    tmp_table_prefix : str, default "tmp_parallel_odps"
        Prefix of the temporary ODPS tables (the staging table of ``pull`` and the chunk tables of ``push``). Must not
        be empty.
    cleanup_tmp : bool, default True
        Drop the temporary ODPS tables when the run ends.
    keep_tmp_on_error : bool, default False
        Keep the temporary tables when the run fails, for debugging. Only relevant when ``cleanup_tmp`` is True.
    """

    unique_key: str | None = None
    chunk_size: int | None = None
    n_chunks: int | None = None
    n_jobs: int = 3
    backend: Backend = "thread"
    pull_split_strategy: PullSplitStrategy = "auto"
    row_number_order_by: str | None = None
    row_number_col: str = "__smf_parallel_odps_rn__"
    validate_unique_key: bool = True
    chunk_filter_key: str = "chunk_filter"
    tmp_dir: Path = field(default_factory=lambda: Path("data/_chunks"))
    tmp_table_prefix: str = "tmp_parallel_odps"
    cleanup_tmp: bool = True
    keep_tmp_on_error: bool = False

    def __post_init__(self) -> None:
        self.tmp_dir = Path(self.tmp_dir)

ParallelODPSManager

Parallel ODPS pull/push helper built on ODPSRunner and ParallelApplyEngine.

Parameters:

Name	Type	Description	Default
config	ParallelODPSConfig	Settings of the manager.	required
odps_runner	ODPSRunner or None	Client used to run the SQL; None creates an ODPSRunner() (which reads its credentials from the environment).	None

Raises:

Type	Description
ValueError	If the configuration is invalid: an unknown backend or pull_split_strategy, a non-positive chunk_size, n_chunks or n_jobs, both chunk_size and n_chunks, or an empty chunk_filter_key, row_number_col or tmp_table_prefix.

Source code in Modeling_Tool/Core/Parallel_ODPS_Manager.py

class ParallelODPSManager:
    """Parallel ODPS pull/push helper built on ODPSRunner and ParallelApplyEngine.

    Parameters
    ----------
    config : ParallelODPSConfig
        Settings of the manager.
    odps_runner : ODPSRunner or None, default None
        Client used to run the SQL; None creates an ``ODPSRunner()`` (which reads its credentials from the environment).

    Raises
    ------
    ValueError
        If the configuration is invalid: an unknown ``backend`` or ``pull_split_strategy``, a non-positive
        ``chunk_size``, ``n_chunks`` or ``n_jobs``, both ``chunk_size`` and ``n_chunks``, or an empty
        ``chunk_filter_key``, ``row_number_col`` or ``tmp_table_prefix``.
    """

    _VALID_BACKENDS = {"thread", "process", "sequential"}
    _VALID_WRITE_MODES = {"overwrite", "append"}
    _VALID_PULL_STRATEGIES = {"auto", "hash", "row_number"}

    def __init__(self, config: ParallelODPSConfig, odps_runner: ODPSRunner | None = None):
        self.config = config
        self.odps_runner = odps_runner or ODPSRunner()
        self._validate_config()

    def _validate_config(self) -> None:
        cfg = self.config
        if cfg.backend not in self._VALID_BACKENDS:
            raise ValueError(f"backend must be one of {sorted(self._VALID_BACKENDS)}")
        if cfg.pull_split_strategy not in self._VALID_PULL_STRATEGIES:
            raise ValueError(f"pull_split_strategy must be one of {sorted(self._VALID_PULL_STRATEGIES)}")
        if cfg.chunk_size is not None and cfg.chunk_size <= 0:
            raise ValueError("chunk_size must be a positive integer.")
        if cfg.n_chunks is not None and cfg.n_chunks <= 0:
            raise ValueError("n_chunks must be a positive integer.")
        if cfg.chunk_size is not None and cfg.n_chunks is not None:
            raise ValueError("chunk_size and n_chunks cannot be used together.")
        if cfg.n_jobs <= 0:
            raise ValueError("n_jobs must be a positive integer.")
        if not cfg.chunk_filter_key:
            raise ValueError("chunk_filter_key cannot be empty.")
        if not cfg.row_number_col:
            raise ValueError("row_number_col cannot be empty.")
        if not cfg.tmp_table_prefix:
            raise ValueError("tmp_table_prefix cannot be empty.")

    def _runner_for_backend(self) -> ODPSRunner | None:
        return None if self.config.backend == "process" else self.odps_runner

    def _auto_count_query(self, sql_path: str, template_kwargs: dict[str, Any]) -> str:
        kwargs = dict(template_kwargs)
        kwargs[self.config.chunk_filter_key] = "1=1"
        rendered_sql = _strip_trailing_semicolon(parse_sql_file(sql_path=sql_path, **kwargs))
        return f"SELECT COUNT(1) FROM ({rendered_sql}) __count_src;"

    def _assert_chunk_filter_placeholder(self, sql_path: str, template_kwargs: dict[str, Any]) -> None:
        sentinel = "__SMF_CHUNK_FILTER_SENTINEL__=1"
        kwargs = dict(template_kwargs)
        kwargs[self.config.chunk_filter_key] = sentinel
        rendered_sql = parse_sql_file(sql_path=sql_path, **kwargs)
        if sentinel not in rendered_sql:
            raise ValueError(
                f"pull SQL template must contain {{{self.config.chunk_filter_key}}}"
            )

    def _render_pull_sql(self, sql_path: str, template_kwargs: dict[str, Any], chunk_filter: str) -> str:
        kwargs = dict(template_kwargs)
        kwargs[self.config.chunk_filter_key] = chunk_filter
        return parse_sql_file(sql_path=sql_path, **kwargs)

    def _resolve_pull_strategy(self) -> str:
        cfg = self.config
        strategy = cfg.pull_split_strategy
        if strategy == "auto":
            return "hash" if cfg.unique_key else "row_number"
        if strategy == "hash" and not cfg.unique_key:
            raise ValueError("unique_key is required when pull_split_strategy='hash'.")
        return strategy

    def _resolve_pull_n_chunks(
        self,
        count_query: str | None,
        sql_path: str,
        template_kwargs: dict[str, Any],
        strategy: str,
        staging_table: str | None = None,
    ) -> int:
        cfg = self.config
        if cfg.n_chunks is not None:
            return cfg.n_chunks
        if cfg.chunk_size is None:
            raise ValueError("chunk_size or n_chunks is required for pull().")
        if count_query is None:
            if strategy == "row_number":
                if not staging_table:
                    raise ValueError("staging_table is required to count row_number pull chunks.")
                count_query = f"SELECT COUNT(1) FROM {staging_table};"
            else:
                count_query = self._auto_count_query(sql_path, template_kwargs)
        count_df = self.odps_runner.run_sql(count_query, to_df=True)
        total_rows = int(count_df.iloc[0, 0])
        return max(1, math.ceil(total_rows / cfg.chunk_size))

    def _hash_chunk_filter(self, chunk_id: int, n_chunks: int) -> str:
        return f"ABS(HASH({self.config.unique_key})) % {n_chunks} = {chunk_id}"

    def _validate_hash_pull_sql(
        self,
        sql_path: str,
        template_kwargs: dict[str, Any],
        n_chunks: int,
    ) -> None:
        if not self.config.validate_unique_key:
            return
        rendered_sql = self._render_pull_sql(
            sql_path=sql_path,
            template_kwargs=template_kwargs,
            chunk_filter=self._hash_chunk_filter(0, n_chunks),
        )
        probe_sql = (
            "SELECT * FROM (\n"
            f"{_strip_trailing_semicolon(rendered_sql)}\n"
            ") __smf_unique_key_probe LIMIT 1;"
        )
        try:
            self.odps_runner.run_sql(probe_sql, to_df=True, n_process=1)
        except Exception as exc:
            raise ValueError(
                f"unique_key validation failed for pull SQL: {_first_error_line(exc)}. "
                f"Common cause: unique_key={self.config.unique_key!r} not visible in "
                f"the SQL scope; see the chained exception for the full ODPS error."
            ) from exc

    def _build_hash_pull_chunks(
        self,
        sql_path: str,
        template_kwargs: dict[str, Any],
        n_chunks: int,
    ) -> list[dict[str, Any]]:
        return [
            {
                "chunk": chunk_id,
                "sql": self._render_pull_sql(
                    sql_path=sql_path,
                    template_kwargs=template_kwargs,
                    chunk_filter=self._hash_chunk_filter(chunk_id, n_chunks),
                ),
            }
            for chunk_id in range(n_chunks)
        ]

    def _row_number_staging_table_name(self, run_id: str) -> str:
        return f"{self.config.tmp_table_prefix}_{run_id}_pull_stage"

    def _create_row_number_staging_table(
        self,
        sql_path: str,
        template_kwargs: dict[str, Any],
        staging_table: str,
    ) -> str:
        cfg = self.config
        base_sql = _strip_trailing_semicolon(
            self._render_pull_sql(
                sql_path=sql_path,
                template_kwargs=template_kwargs,
                chunk_filter="1=1",
            )
        )
        order_by = cfg.row_number_order_by or "1"
        create_sql = (
            f"CREATE TABLE {staging_table} AS\n"
            "SELECT\n"
            f"  ROW_NUMBER() OVER (ORDER BY {order_by}) AS {cfg.row_number_col},\n"
            "  __smf_base.*\n"
            "FROM (\n"
            f"{base_sql}\n"
            ") __smf_base;"
        )
        self.odps_runner.run_sql(create_sql, to_df=False)
        return create_sql

    def _build_row_number_pull_chunks(self, staging_table: str, n_chunks: int) -> list[dict[str, Any]]:
        row_number_col = self.config.row_number_col
        return [
            {
                "chunk": chunk_id,
                "sql": (
                    f"SELECT *\n"
                    f"FROM {staging_table}\n"
                    f"WHERE ({row_number_col} - 1) % {n_chunks} = {chunk_id};"
                ),
                "drop_cols": [row_number_col],
            }
            for chunk_id in range(n_chunks)
        ]

    def _run_pull_chunks(self, chunk_specs: list[dict[str, Any]]) -> list[dict[str, Any]]:
        cfg = self.config
        engine_cfg = ParallelApplyConfig(
            split_axis="chunk",
            backend=cfg.backend,
            n_jobs=cfg.n_jobs,
            combine="list",
            on_error="collect",
        )
        result = ParallelApplyEngine(engine_cfg).run(
            func=_pull_one_chunk,
            chunks=chunk_specs,
            func_args=(
                self._runner_for_backend(),
                str(cfg.tmp_dir),
            ),
        )
        if len(result.errors):
            raise RuntimeError(f"{len(result.errors)}/{len(chunk_specs)} ODPS pull chunks failed:\n{result.errors}")
        return sorted(result.output, key=lambda item: item["chunk"])

    def pull(
        self,
        sql_path: str,
        out_path: str,
        count_query: str | None = None,
        **template_kwargs: Any,
    ) -> dict[str, Any]:
        """Pull a large query in concurrent chunks and merge them into one local CSV.

        Parameters
        ----------
        sql_path : str
            Path of a SQL template file. It must contain the ``{chunk_filter}`` placeholder (see ``chunk_filter_key``)
            in the ``WHERE`` clause of the table to split.
        out_path : str
            Local CSV file to write. Its folder is created and an existing file is replaced.
        count_query : str or None, default None
            SQL that returns the total row count in its first cell. It is only used when ``chunk_size`` is set (to
            compute the number of chunks); None counts the rows of the query automatically.
        **template_kwargs
            Values for the other placeholders of the SQL template.

        Returns
        -------
        dict
            ``pull_strategy``, ``staging_table`` (None for hash pulls), ``n_chunks``, ``total_rows``, ``out_path`` and
            ``per_chunk_rows``.

        Raises
        ------
        ValueError
            If the template lacks the placeholder, neither ``chunk_size`` nor ``n_chunks`` is configured, hash mode has
            no ``unique_key``, or the ``unique_key`` probe fails.
        """
        cfg = self.config
        self._assert_chunk_filter_placeholder(sql_path, template_kwargs)
        cfg.tmp_dir.mkdir(parents=True, exist_ok=True)
        strategy = self._resolve_pull_strategy()
        staging_table: str | None = None
        success = False

        try:
            if strategy == "row_number":
                run_id = uuid.uuid4().hex[:12]
                staging_table = self._row_number_staging_table_name(run_id)
                self._create_row_number_staging_table(sql_path, template_kwargs, staging_table)
                n_chunks = self._resolve_pull_n_chunks(
                    count_query=count_query,
                    sql_path=sql_path,
                    template_kwargs=template_kwargs,
                    strategy=strategy,
                    staging_table=staging_table,
                )
                chunk_specs = self._build_row_number_pull_chunks(staging_table, n_chunks)
            else:
                n_chunks = self._resolve_pull_n_chunks(
                    count_query=count_query,
                    sql_path=sql_path,
                    template_kwargs=template_kwargs,
                    strategy=strategy,
                )
                self._validate_hash_pull_sql(sql_path, template_kwargs, n_chunks)
                chunk_specs = self._build_hash_pull_chunks(sql_path, template_kwargs, n_chunks)

            chunk_summaries = self._run_pull_chunks(chunk_specs)
            final_path = Path(out_path)
            final_path.parent.mkdir(parents=True, exist_ok=True)
            final_path.unlink(missing_ok=True)

            with open(final_path, "wb") as fout:
                for idx, summary in enumerate(chunk_summaries):
                    chunk_file = Path(summary["path"])
                    with open(chunk_file, "rb") as fin:
                        if idx > 0:
                            fin.readline()
                        shutil.copyfileobj(fin, fout)
                    chunk_file.unlink()

            success = True
            return {
                "pull_strategy": strategy,
                "staging_table": staging_table,
                "n_chunks": n_chunks,
                "total_rows": sum(item["rows"] for item in chunk_summaries),
                "out_path": str(final_path),
                "per_chunk_rows": [item["rows"] for item in chunk_summaries],
            }
        finally:
            if (
                staging_table
                and cfg.cleanup_tmp
                and (success or not cfg.keep_tmp_on_error)
            ):
                _delete_table(self.odps_runner, staging_table)

    def push(
        self,
        data: pd.DataFrame | str | Path,
        target_table: str,
        write_mode: WriteMode | str | None = None,
    ) -> dict[str, Any]:
        """Upload a DataFrame or CSV file in concurrent chunks and write them into one ODPS table.

        Every chunk is uploaded to a temporary table; the target table is then created or filled from all of them with
        ``UNION ALL``.

        Parameters
        ----------
        data : pandas.DataFrame, str or pathlib.Path
            The rows to upload, or the path of a CSV file (read in chunks and copied to ``tmp_dir``).
        target_table : str
            Name of the table to write.
        write_mode : {"overwrite", "append"}, default None
            Required despite the default None. ``"overwrite"`` drops the target table and re-creates it from the chunks
            (its schema, partitions and properties are replaced); ``"append"`` runs ``INSERT INTO TABLE``.

        Returns
        -------
        dict
            ``n_chunks``, ``total_rows``, ``target_table``, ``write_mode``, ``tmp_tables``, ``per_chunk_rows``,
            ``union_sql`` and ``final_sql``.

        Raises
        ------
        ValueError
            If ``write_mode`` is missing or invalid, ``target_table`` is empty, no chunk size is configured, or the data
            has no columns.
        TypeError
            If ``data`` is neither a DataFrame nor a path.
        FileNotFoundError
            If a CSV path does not exist.
        RuntimeError
            If any chunk fails to upload.
        """
        if write_mode not in self._VALID_WRITE_MODES:
            raise ValueError("write_mode is required and must be one of ['append', 'overwrite'].")
        if not target_table:
            raise ValueError("target_table cannot be empty.")

        cfg = self.config
        cfg.tmp_dir.mkdir(parents=True, exist_ok=True)
        run_id = uuid.uuid4().hex[:12]
        chunk_specs, local_temp_paths = self._build_push_chunks(data=data, run_id=run_id)
        if not chunk_specs:
            raise ValueError("push data produced no chunks.")

        tmp_tables = [spec["tmp_table"] for spec in chunk_specs]
        success = False
        try:
            engine_cfg = ParallelApplyConfig(
                split_axis="chunk",
                backend=cfg.backend,
                n_jobs=cfg.n_jobs,
                combine="list",
                on_error="collect",
            )
            result = ParallelApplyEngine(engine_cfg).run(
                func=_push_one_chunk,
                chunks=chunk_specs,
                func_args=(self._runner_for_backend(),),
            )
            if len(result.errors):
                raise RuntimeError(f"{len(result.errors)}/{len(chunk_specs)} ODPS push chunks failed:\n{result.errors}")

            chunk_summaries = sorted(result.output, key=lambda item: item["chunk"])
            union_sql = self._build_union_sql(tmp_tables)
            final_sql = self._write_final_table(target_table=target_table, union_sql=union_sql, write_mode=str(write_mode))

            success = True
            return {
                "n_chunks": len(chunk_summaries),
                "total_rows": sum(item["rows"] for item in chunk_summaries),
                "target_table": target_table,
                "write_mode": write_mode,
                "tmp_tables": tmp_tables,
                "per_chunk_rows": [item["rows"] for item in chunk_summaries],
                "union_sql": union_sql,
                "final_sql": final_sql,
            }
        finally:
            self._cleanup_local_files(local_temp_paths)
            if cfg.cleanup_tmp and (success or not cfg.keep_tmp_on_error):
                self._cleanup_tmp_tables(tmp_tables)

    def _build_push_chunks(self, data: pd.DataFrame | str | Path, run_id: str) -> tuple[list[dict[str, Any]], list[Path]]:
        if isinstance(data, pd.DataFrame):
            return self._build_dataframe_push_chunks(data, run_id), []
        if isinstance(data, (str, Path)):
            return self._build_csv_push_chunks(Path(data), run_id)
        raise TypeError("data must be a pandas DataFrame or a CSV path.")

    def _build_dataframe_push_chunks(self, data: pd.DataFrame, run_id: str) -> list[dict[str, Any]]:
        if len(data.columns) == 0:
            raise ValueError("data must contain at least one column.")
        positions = self._split_positions(len(data))
        specs = []
        for chunk_id, pos in enumerate(positions):
            chunk_df = data.iloc[pos].copy() if len(pos) else data.iloc[0:0].copy()
            specs.append({
                "chunk": chunk_id,
                "data": chunk_df,
                "tmp_table": self._tmp_table_name(run_id, chunk_id),
            })
        return specs

    def _build_csv_push_chunks(self, csv_path: Path, run_id: str) -> tuple[list[dict[str, Any]], list[Path]]:
        if not csv_path.is_file():
            raise FileNotFoundError(f"CSV file not found: {csv_path}")
        chunk_size = self._resolve_csv_chunk_size(csv_path)
        specs: list[dict[str, Any]] = []
        local_paths: list[Path] = []
        for chunk_id, chunk_df in enumerate(pd.read_csv(csv_path, chunksize=chunk_size)):
            local_path = self.config.tmp_dir / f"_push_{run_id}_{chunk_id:04d}.csv"
            chunk_df.to_csv(local_path, index=False)
            local_paths.append(local_path)
            specs.append({
                "chunk": chunk_id,
                "csv_path": str(local_path),
                "tmp_table": self._tmp_table_name(run_id, chunk_id),
            })
        return specs, local_paths

    def _resolve_csv_chunk_size(self, csv_path: Path) -> int:
        cfg = self.config
        if cfg.chunk_size is not None:
            return cfg.chunk_size
        if cfg.n_chunks is None:
            raise ValueError("chunk_size or n_chunks is required for CSV push().")
        with open(csv_path, "rb") as fin:
            n_lines = sum(1 for _ in fin)
        n_rows = max(0, n_lines - 1)
        return max(1, math.ceil(n_rows / cfg.n_chunks))

    def _split_positions(self, n_rows: int) -> list[np.ndarray]:
        cfg = self.config
        if n_rows < 0:
            raise ValueError("n_rows must be non-negative.")
        if n_rows == 0:
            return [np.array([], dtype=int)]
        if cfg.chunk_size is not None:
            n_chunks = math.ceil(n_rows / cfg.chunk_size)
        elif cfg.n_chunks is not None:
            n_chunks = min(cfg.n_chunks, n_rows)
        else:
            raise ValueError("chunk_size or n_chunks is required for push().")
        return [arr for arr in np.array_split(np.arange(n_rows), n_chunks) if len(arr) > 0]

    def _tmp_table_name(self, run_id: str, chunk_id: int) -> str:
        return f"{self.config.tmp_table_prefix}_{run_id}_{chunk_id:04d}"

    @staticmethod
    def _build_union_sql(tmp_tables: list[str]) -> str:
        return "\nUNION ALL\n".join(f"SELECT * FROM {table_name}" for table_name in tmp_tables)

    def _write_final_table(self, target_table: str, union_sql: str, write_mode: str) -> str:
        if write_mode == "overwrite":
            _delete_table(self.odps_runner, target_table)
            final_sql = f"CREATE TABLE {target_table} AS\n{union_sql};"
        else:
            final_sql = f"INSERT INTO TABLE {target_table}\n{union_sql};"
        self.odps_runner.run_sql(final_sql, to_df=False)
        return final_sql

    def _cleanup_tmp_tables(self, tmp_tables: list[str]) -> None:
        for table_name in tmp_tables:
            _delete_table(self.odps_runner, table_name)

    @staticmethod
    def _cleanup_local_files(paths: list[Path]) -> None:
        for path in paths:
            path.unlink(missing_ok=True)

pull

pull(sql_path: str, out_path: str, count_query: str | None = None, **template_kwargs: Any) -> dict[str, Any]

Pull a large query in concurrent chunks and merge them into one local CSV.

Parameters:

Name	Type	Description	Default
sql_path	str	Path of a SQL template file. It must contain the {chunk_filter} placeholder (see chunk_filter_key) in the WHERE clause of the table to split.	required
out_path	str	Local CSV file to write. Its folder is created and an existing file is replaced.	required
count_query	str or None	SQL that returns the total row count in its first cell. It is only used when chunk_size is set (to compute the number of chunks); None counts the rows of the query automatically.	None
**template_kwargs	Any	Values for the other placeholders of the SQL template.	{}

Returns:

Type	Description
dict	pull_strategy, staging_table (None for hash pulls), n_chunks, total_rows, out_path and per_chunk_rows.

Raises:

Type	Description
ValueError	If the template lacks the placeholder, neither chunk_size nor n_chunks is configured, hash mode has no unique_key, or the unique_key probe fails.

Source code in Modeling_Tool/Core/Parallel_ODPS_Manager.py

def pull(
    self,
    sql_path: str,
    out_path: str,
    count_query: str | None = None,
    **template_kwargs: Any,
) -> dict[str, Any]:
    """Pull a large query in concurrent chunks and merge them into one local CSV.

    Parameters
    ----------
    sql_path : str
        Path of a SQL template file. It must contain the ``{chunk_filter}`` placeholder (see ``chunk_filter_key``)
        in the ``WHERE`` clause of the table to split.
    out_path : str
        Local CSV file to write. Its folder is created and an existing file is replaced.
    count_query : str or None, default None
        SQL that returns the total row count in its first cell. It is only used when ``chunk_size`` is set (to
        compute the number of chunks); None counts the rows of the query automatically.
    **template_kwargs
        Values for the other placeholders of the SQL template.

    Returns
    -------
    dict
        ``pull_strategy``, ``staging_table`` (None for hash pulls), ``n_chunks``, ``total_rows``, ``out_path`` and
        ``per_chunk_rows``.

    Raises
    ------
    ValueError
        If the template lacks the placeholder, neither ``chunk_size`` nor ``n_chunks`` is configured, hash mode has
        no ``unique_key``, or the ``unique_key`` probe fails.
    """
    cfg = self.config
    self._assert_chunk_filter_placeholder(sql_path, template_kwargs)
    cfg.tmp_dir.mkdir(parents=True, exist_ok=True)
    strategy = self._resolve_pull_strategy()
    staging_table: str | None = None
    success = False

    try:
        if strategy == "row_number":
            run_id = uuid.uuid4().hex[:12]
            staging_table = self._row_number_staging_table_name(run_id)
            self._create_row_number_staging_table(sql_path, template_kwargs, staging_table)
            n_chunks = self._resolve_pull_n_chunks(
                count_query=count_query,
                sql_path=sql_path,
                template_kwargs=template_kwargs,
                strategy=strategy,
                staging_table=staging_table,
            )
            chunk_specs = self._build_row_number_pull_chunks(staging_table, n_chunks)
        else:
            n_chunks = self._resolve_pull_n_chunks(
                count_query=count_query,
                sql_path=sql_path,
                template_kwargs=template_kwargs,
                strategy=strategy,
            )
            self._validate_hash_pull_sql(sql_path, template_kwargs, n_chunks)
            chunk_specs = self._build_hash_pull_chunks(sql_path, template_kwargs, n_chunks)

        chunk_summaries = self._run_pull_chunks(chunk_specs)
        final_path = Path(out_path)
        final_path.parent.mkdir(parents=True, exist_ok=True)
        final_path.unlink(missing_ok=True)

        with open(final_path, "wb") as fout:
            for idx, summary in enumerate(chunk_summaries):
                chunk_file = Path(summary["path"])
                with open(chunk_file, "rb") as fin:
                    if idx > 0:
                        fin.readline()
                    shutil.copyfileobj(fin, fout)
                chunk_file.unlink()

        success = True
        return {
            "pull_strategy": strategy,
            "staging_table": staging_table,
            "n_chunks": n_chunks,
            "total_rows": sum(item["rows"] for item in chunk_summaries),
            "out_path": str(final_path),
            "per_chunk_rows": [item["rows"] for item in chunk_summaries],
        }
    finally:
        if (
            staging_table
            and cfg.cleanup_tmp
            and (success or not cfg.keep_tmp_on_error)
        ):
            _delete_table(self.odps_runner, staging_table)

push

push(data: DataFrame | str | Path, target_table: str, write_mode: WriteMode | str | None = None) -> dict[str, Any]

Upload a DataFrame or CSV file in concurrent chunks and write them into one ODPS table.

Every chunk is uploaded to a temporary table; the target table is then created or filled from all of them with UNION ALL.

Parameters:

Name	Type	Description	Default
data	(DataFrame, str or Path)	The rows to upload, or the path of a CSV file (read in chunks and copied to tmp_dir).	required
target_table	str	Name of the table to write.	required
write_mode	('overwrite', 'append')	Required despite the default None. "overwrite" drops the target table and re-creates it from the chunks (its schema, partitions and properties are replaced); "append" runs INSERT INTO TABLE.	"overwrite"

Returns:

Type	Description
dict	n_chunks, total_rows, target_table, write_mode, tmp_tables, per_chunk_rows, union_sql and final_sql.

Raises:

Type	Description
ValueError	If write_mode is missing or invalid, target_table is empty, no chunk size is configured, or the data has no columns.
TypeError	If data is neither a DataFrame nor a path.
FileNotFoundError	If a CSV path does not exist.
RuntimeError	If any chunk fails to upload.

Source code in Modeling_Tool/Core/Parallel_ODPS_Manager.py

def push(
    self,
    data: pd.DataFrame | str | Path,
    target_table: str,
    write_mode: WriteMode | str | None = None,
) -> dict[str, Any]:
    """Upload a DataFrame or CSV file in concurrent chunks and write them into one ODPS table.

    Every chunk is uploaded to a temporary table; the target table is then created or filled from all of them with
    ``UNION ALL``.

    Parameters
    ----------
    data : pandas.DataFrame, str or pathlib.Path
        The rows to upload, or the path of a CSV file (read in chunks and copied to ``tmp_dir``).
    target_table : str
        Name of the table to write.
    write_mode : {"overwrite", "append"}, default None
        Required despite the default None. ``"overwrite"`` drops the target table and re-creates it from the chunks
        (its schema, partitions and properties are replaced); ``"append"`` runs ``INSERT INTO TABLE``.

    Returns
    -------
    dict
        ``n_chunks``, ``total_rows``, ``target_table``, ``write_mode``, ``tmp_tables``, ``per_chunk_rows``,
        ``union_sql`` and ``final_sql``.

    Raises
    ------
    ValueError
        If ``write_mode`` is missing or invalid, ``target_table`` is empty, no chunk size is configured, or the data
        has no columns.
    TypeError
        If ``data`` is neither a DataFrame nor a path.
    FileNotFoundError
        If a CSV path does not exist.
    RuntimeError
        If any chunk fails to upload.
    """
    if write_mode not in self._VALID_WRITE_MODES:
        raise ValueError("write_mode is required and must be one of ['append', 'overwrite'].")
    if not target_table:
        raise ValueError("target_table cannot be empty.")

    cfg = self.config
    cfg.tmp_dir.mkdir(parents=True, exist_ok=True)
    run_id = uuid.uuid4().hex[:12]
    chunk_specs, local_temp_paths = self._build_push_chunks(data=data, run_id=run_id)
    if not chunk_specs:
        raise ValueError("push data produced no chunks.")

    tmp_tables = [spec["tmp_table"] for spec in chunk_specs]
    success = False
    try:
        engine_cfg = ParallelApplyConfig(
            split_axis="chunk",
            backend=cfg.backend,
            n_jobs=cfg.n_jobs,
            combine="list",
            on_error="collect",
        )
        result = ParallelApplyEngine(engine_cfg).run(
            func=_push_one_chunk,
            chunks=chunk_specs,
            func_args=(self._runner_for_backend(),),
        )
        if len(result.errors):
            raise RuntimeError(f"{len(result.errors)}/{len(chunk_specs)} ODPS push chunks failed:\n{result.errors}")

        chunk_summaries = sorted(result.output, key=lambda item: item["chunk"])
        union_sql = self._build_union_sql(tmp_tables)
        final_sql = self._write_final_table(target_table=target_table, union_sql=union_sql, write_mode=str(write_mode))

        success = True
        return {
            "n_chunks": len(chunk_summaries),
            "total_rows": sum(item["rows"] for item in chunk_summaries),
            "target_table": target_table,
            "write_mode": write_mode,
            "tmp_tables": tmp_tables,
            "per_chunk_rows": [item["rows"] for item in chunk_summaries],
            "union_sql": union_sql,
            "final_sql": final_sql,
        }
    finally:
        self._cleanup_local_files(local_temp_paths)
        if cfg.cleanup_tmp and (success or not cfg.keep_tmp_on_error):
            self._cleanup_tmp_tables(tmp_tables)

Parallel Apply Engine: Parallel_Engine

Parallel_Engine

ParallelApplyConfig dataclass

Settings of a ParallelApplyEngine run.

Parameters:

Name	Type	Description	Default
split_axis	('row', 'column', 'chunk', 'auto')	How the work is divided. "row" splits the DataFrame into row blocks. "column" splits it into column blocks, each of which also carries id_cols and required_cols. "chunk" calls the function once for every object of the chunks argument of run. "auto" applies the function to a sample of at most 20 rows, once whole and once in row blocks and in column blocks, and uses the axis whose combined result equals the whole result; it raises ValueError when none or both match.	"row"
backend	('process', 'thread', 'sequential')	"process" uses a joblib (loky) process pool, "thread" uses threads, and "sequential" runs the chunks one after another in the calling process. One resolved job (or one chunk) also runs sequentially.	"process"
n_jobs	(int, str or None)	Number of workers: a positive integer, -1 for all CPU cores, "auto" for all cores but one, or None for a single worker. It never exceeds the number of chunks.	"auto"
chunk_size	int or None	Rows per row chunk, or columns per column chunk (not counting id_cols and required_cols). Cannot be combined with n_chunks.	None
n_chunks	int or None	Number of chunks to make (at most the number of rows or columns). Cannot be combined with chunk_size. With neither set, one chunk per worker is made.	None
preserve_order	bool	Sort the chunk results by chunk id before they are combined.	True
combine	('concat', 'list', 'dict', 'none')	How the chunk outputs become output: "concat" joins them with pandas.concat, "list" returns the list of outputs, "dict" returns {chunk_id: output}, and "none" returns None.	"concat"
concat_axis	(0, 1)	Axis of the pandas.concat call; None means 1 for column splits and 0 otherwise.	0
pass_chunk_info	bool	When True the function also receives a chunk_info keyword argument, a dict with chunk_id, rows and columns (None for custom chunks).	False
timeout	float or None	Seconds to wait for the workers, passed to joblib. It has no effect on sequential runs.	None
on_error	('raise', 'collect')	"raise" stops with the first exception. "collect" records each failed chunk in errors (columns chunk_id, error_type, error_message and traceback) and combines the successful chunks.	"raise"
validate_picklable	bool	With the process backend, check with cloudpickle before the run that the function and its arguments can be serialized, and raise TypeError if they cannot.	True
auto_probe	bool	Accepted but not used by the engine.	False
random_state	int or None	Seed of the row sample that the split_axis="auto" probe draws.	None
required_cols	list of str	Columns added to every block of a column split.	empty list
id_cols	list of str	Identifier columns added to every block of a column split (same effect as required_cols).	empty list

Source code in Modeling_Tool/Core/Parallel_Engine.py

@dataclass
class ParallelApplyConfig:
    """Settings of a ``ParallelApplyEngine`` run.

    Parameters
    ----------
    split_axis : {"row", "column", "chunk", "auto"}, default "row"
        How the work is divided. ``"row"`` splits the DataFrame into row blocks. ``"column"`` splits it into column
        blocks, each of which also carries ``id_cols`` and ``required_cols``. ``"chunk"`` calls the function once for
        every object of the ``chunks`` argument of ``run``. ``"auto"`` applies the function to a sample of at most 20
        rows, once whole and once in row blocks and in column blocks, and uses the axis whose combined result equals the
        whole result; it raises ``ValueError`` when none or both match.
    backend : {"process", "thread", "sequential"}, default "process"
        ``"process"`` uses a joblib (loky) process pool, ``"thread"`` uses threads, and ``"sequential"`` runs the chunks
        one after another in the calling process. One resolved job (or one chunk) also runs sequentially.
    n_jobs : int, str or None, default "auto"
        Number of workers: a positive integer, ``-1`` for all CPU cores, ``"auto"`` for all cores but one, or None for a
        single worker. It never exceeds the number of chunks.
    chunk_size : int or None, default None
        Rows per row chunk, or columns per column chunk (not counting ``id_cols`` and ``required_cols``). Cannot be
        combined with ``n_chunks``.
    n_chunks : int or None, default None
        Number of chunks to make (at most the number of rows or columns). Cannot be combined with ``chunk_size``. With
        neither set, one chunk per worker is made.
    preserve_order : bool, default True
        Sort the chunk results by chunk id before they are combined.
    combine : {"concat", "list", "dict", "none"}, default "concat"
        How the chunk outputs become ``output``: ``"concat"`` joins them with ``pandas.concat``, ``"list"`` returns the
        list of outputs, ``"dict"`` returns ``{chunk_id: output}``, and ``"none"`` returns None.
    concat_axis : {0, 1} or None, default None
        Axis of the ``pandas.concat`` call; None means 1 for column splits and 0 otherwise.
    pass_chunk_info : bool, default False
        When True the function also receives a ``chunk_info`` keyword argument, a dict with ``chunk_id``, ``rows`` and
        ``columns`` (None for custom chunks).
    timeout : float or None, default None
        Seconds to wait for the workers, passed to joblib. It has no effect on sequential runs.
    on_error : {"raise", "collect"}, default "raise"
        ``"raise"`` stops with the first exception. ``"collect"`` records each failed chunk in ``errors`` (columns
        ``chunk_id``, ``error_type``, ``error_message`` and ``traceback``) and combines the successful chunks.
    validate_picklable : bool, default True
        With the process backend, check with cloudpickle before the run that the function and its arguments can be
        serialized, and raise ``TypeError`` if they cannot.
    auto_probe : bool, default False
        Accepted but not used by the engine.
    random_state : int or None, default None
        Seed of the row sample that the ``split_axis="auto"`` probe draws.
    required_cols : list of str, default empty list
        Columns added to every block of a column split.
    id_cols : list of str, default empty list
        Identifier columns added to every block of a column split (same effect as ``required_cols``).
    """

    split_axis: SplitAxis = "row"
    backend: Backend = "process"
    n_jobs: int | str | None = "auto"
    chunk_size: int | None = None
    n_chunks: int | None = None
    preserve_order: bool = True
    combine: CombineMode = "concat"
    concat_axis: Literal[0, 1] | None = None
    pass_chunk_info: bool = False
    timeout: float | None = None
    on_error: ErrorMode = "raise"
    validate_picklable: bool = True
    auto_probe: bool = False
    random_state: int | None = None
    required_cols: list[str] = field(default_factory=list)
    id_cols: list[str] = field(default_factory=list)

ParallelApplyResult dataclass

Result of ParallelApplyEngine.run.

Parameters:

Name	Type	Description	Default
output	object	The combined output, as selected by ParallelApplyConfig.combine.	required
chunk_outputs	list	Outputs of the successful chunks (in chunk order when preserve_order is True).	empty list
errors	DataFrame	One row per failed chunk when on_error="collect"; empty otherwise.	empty frame
summary	dict	Run statistics: split_axis, backend, n_jobs, n_chunks, n_success, n_error and elapsed_seconds.	empty dict
config	ParallelApplyConfig or None	The configuration the run used.	None
split_axis_resolved	str or None	The axis that was used ("row", "column" or "chunk"); this is how to read the outcome of split_axis="auto".	None

Source code in Modeling_Tool/Core/Parallel_Engine.py

@dataclass
class ParallelApplyResult:
    """Result of ``ParallelApplyEngine.run``.

    Parameters
    ----------
    output : object
        The combined output, as selected by ``ParallelApplyConfig.combine``.
    chunk_outputs : list, default empty list
        Outputs of the successful chunks (in chunk order when ``preserve_order`` is True).
    errors : pandas.DataFrame, default empty frame
        One row per failed chunk when ``on_error="collect"``; empty otherwise.
    summary : dict, default empty dict
        Run statistics: ``split_axis``, ``backend``, ``n_jobs``, ``n_chunks``, ``n_success``, ``n_error`` and
        ``elapsed_seconds``.
    config : ParallelApplyConfig or None, default None
        The configuration the run used.
    split_axis_resolved : str or None, default None
        The axis that was used (``"row"``, ``"column"`` or ``"chunk"``); this is how to read the outcome of
        ``split_axis="auto"``.
    """

    output: Any
    chunk_outputs: list[Any] = field(default_factory=list)
    errors: pd.DataFrame = field(default_factory=pd.DataFrame)
    summary: dict[str, Any] = field(default_factory=dict)
    config: ParallelApplyConfig | None = None
    split_axis_resolved: str | None = None

ParallelApplyEngine

Parallel function dispatcher for DataFrame and custom chunk workloads.

Parameters:

Name	Type	Description	Default
config	ParallelApplyConfig or None	Settings of the engine; None uses ParallelApplyConfig().	None

Raises:

Type	Description
ValueError	If a setting is invalid: an unknown split_axis, backend, combine or on_error, a non-positive chunk_size or n_chunks, both chunk_size and n_chunks, or a concat_axis other than 0, 1 or None.

Source code in Modeling_Tool/Core/Parallel_Engine.py

class ParallelApplyEngine:
    """Parallel function dispatcher for DataFrame and custom chunk workloads.

    Parameters
    ----------
    config : ParallelApplyConfig or None, default None
        Settings of the engine; None uses ``ParallelApplyConfig()``.

    Raises
    ------
    ValueError
        If a setting is invalid: an unknown ``split_axis``, ``backend``, ``combine`` or ``on_error``, a non-positive
        ``chunk_size`` or ``n_chunks``, both ``chunk_size`` and ``n_chunks``, or a ``concat_axis`` other than 0, 1 or
        None.
    """

    _VALID_SPLIT_AXIS = {"row", "column", "chunk", "auto"}
    _VALID_BACKENDS = {"process", "thread", "sequential"}
    _VALID_COMBINE = {"concat", "list", "dict", "none"}
    _VALID_ON_ERROR = {"raise", "collect"}

    def __init__(self, config: ParallelApplyConfig | None = None):
        self.config = config or ParallelApplyConfig()
        self._validate_config(self.config)

    def run(
        self,
        data: Any = None,
        func: Callable[..., Any] | None = None,
        func_args: Sequence[Any] = (),
        func_kwargs: dict[str, Any] | None = None,
        chunks: Sequence[Any] | None = None,
    ) -> ParallelApplyResult:
        """Split the work, run ``func`` on every chunk, and combine the outputs.

        Parameters
        ----------
        data : pandas.DataFrame or None, default None
            Frame to split. It must be a DataFrame unless ``split_axis="chunk"``, and also with ``split_axis="auto"``.
        func : callable
            Called as ``func(chunk, *func_args, **func_kwargs)`` for every chunk, with an extra ``chunk_info`` keyword
            argument when ``pass_chunk_info`` is True. It is required although the signature default is None.
        func_args : sequence, default ()
            Extra positional arguments for ``func``.
        func_kwargs : dict or None, default None
            Extra keyword arguments for ``func``.
        chunks : sequence or None, default None
            The objects to process one by one; required when ``split_axis="chunk"``. With ``split_axis="auto"`` a
            given ``chunks`` selects the chunk mode.

        Returns
        -------
        ParallelApplyResult
            The combined ``output``, the per-chunk outputs, the collected ``errors`` and a ``summary``.

        Raises
        ------
        ValueError
            If ``func`` is None, ``split_axis="chunk"`` has no ``chunks``, ``split_axis="auto"`` cannot decide on a
            single axis, or ``n_jobs`` is invalid.
        TypeError
            If ``data`` is not a DataFrame where one is required, or if ``func`` and its arguments cannot be serialized
            for the process backend.
        KeyError
            If ``required_cols`` or ``id_cols`` are missing from ``data`` in a column split.
        """
        if func is None:
            raise ValueError("func is required.")

        cfg = self.config
        func_kwargs = {} if func_kwargs is None else dict(func_kwargs)
        func_args = tuple(func_args)
        self._validate_runtime_inputs(data=data, chunks=chunks)
        if cfg.validate_picklable and cfg.backend == "process":
            self._validate_picklable(func, func_args, func_kwargs)

        start = time.time()
        split_axis = self._resolve_split_axis(data, func, func_args, func_kwargs, chunks)
        chunk_specs = self._make_chunks(data=data, chunks=chunks, split_axis=split_axis)
        n_jobs = self._resolve_n_jobs(cfg.n_jobs, len(chunk_specs))

        raw_results = self._execute(
            func=func,
            func_args=func_args,
            func_kwargs=func_kwargs,
            chunk_specs=chunk_specs,
            n_jobs=n_jobs,
        )
        if cfg.preserve_order:
            raw_results = sorted(raw_results, key=lambda item: item["chunk_id"])

        successful = [item for item in raw_results if item["ok"]]
        chunk_outputs = [item["output"] for item in successful]
        errors = pd.DataFrame([item["error"] for item in raw_results if not item["ok"]])
        output = self._combine_outputs(successful, split_axis)
        elapsed = time.time() - start

        return ParallelApplyResult(
            output=output,
            chunk_outputs=chunk_outputs,
            errors=errors,
            summary={
                "split_axis": split_axis,
                "backend": cfg.backend,
                "n_jobs": n_jobs,
                "n_chunks": len(chunk_specs),
                "n_success": len(successful),
                "n_error": int(len(errors)),
                "elapsed_seconds": elapsed,
            },
            config=cfg,
            split_axis_resolved=split_axis,
        )

    def _validate_config(self, cfg: ParallelApplyConfig) -> None:
        if cfg.split_axis not in self._VALID_SPLIT_AXIS:
            raise ValueError(f"split_axis must be one of {sorted(self._VALID_SPLIT_AXIS)}")
        if cfg.backend not in self._VALID_BACKENDS:
            raise ValueError(f"backend must be one of {sorted(self._VALID_BACKENDS)}")
        if cfg.combine not in self._VALID_COMBINE:
            raise ValueError(f"combine must be one of {sorted(self._VALID_COMBINE)}")
        if cfg.on_error not in self._VALID_ON_ERROR:
            raise ValueError(f"on_error must be one of {sorted(self._VALID_ON_ERROR)}")
        if cfg.chunk_size is not None and cfg.chunk_size <= 0:
            raise ValueError("chunk_size must be a positive integer.")
        if cfg.n_chunks is not None and cfg.n_chunks <= 0:
            raise ValueError("n_chunks must be a positive integer.")
        if cfg.chunk_size is not None and cfg.n_chunks is not None:
            raise ValueError("chunk_size and n_chunks cannot be used together.")
        if cfg.concat_axis is not None and cfg.concat_axis not in {0, 1}:
            raise ValueError("concat_axis must be 0, 1, or None.")

    def _validate_runtime_inputs(self, data: Any, chunks: Sequence[Any] | None) -> None:
        cfg = self.config
        if cfg.split_axis in {"row", "column", "auto"} and not isinstance(data, pd.DataFrame):
            raise TypeError("data must be a pandas DataFrame when split_axis is row, column, or auto.")
        if cfg.split_axis == "chunk" and chunks is None:
            raise ValueError("chunks is required when split_axis='chunk'.")

    def _validate_picklable(
        self,
        func: Callable[..., Any],
        func_args: tuple[Any, ...],
        func_kwargs: dict[str, Any],
    ) -> None:
        # joblib < 1.6 vendors cloudpickle; joblib >= 1.6 dropped the copy and depends on
        # the cloudpickle package. Resolve the serializer outside the check below so an
        # import problem is never reported as a non-serializable callable.
        try:
            from joblib.externals import cloudpickle
        except ImportError:
            import cloudpickle

        try:
            cloudpickle.dumps((func, func_args, func_kwargs))
        except Exception as exc:
            raise TypeError(
                "func, func_args, and func_kwargs must be serializable for backend='process'. "
                "Use backend='thread' or backend='sequential' for non-serializable callables."
            ) from exc

    def _resolve_split_axis(
        self,
        data: Any,
        func: Callable[..., Any],
        func_args: tuple[Any, ...],
        func_kwargs: dict[str, Any],
        chunks: Sequence[Any] | None,
    ) -> str:
        cfg = self.config
        if cfg.split_axis != "auto":
            return cfg.split_axis
        if chunks is not None:
            return "chunk"
        return self._probe_split_axis(data, func, func_args, func_kwargs)

    def _probe_split_axis(
        self,
        data: pd.DataFrame,
        func: Callable[..., Any],
        func_args: tuple[Any, ...],
        func_kwargs: dict[str, Any],
    ) -> str:
        sample = data.sample(
            n=min(len(data), 20),
            random_state=self.config.random_state,
        ) if len(data) > 20 else data.copy()
        full_output = func(sample.copy(), *func_args, **func_kwargs)
        candidates: list[str] = []
        for axis in ("row", "column"):
            try:
                chunk_specs = self._make_chunks(data=sample, chunks=None, split_axis=axis, force_n_chunks=2)
                outputs = []
                for spec in chunk_specs:
                    outputs.append(func(spec.chunk, *func_args, **func_kwargs))
                probe_results = [
                    {"chunk_id": i, "output": output}
                    for i, output in enumerate(outputs)
                ]
                combined = self._combine_outputs(probe_results, axis)
                if self._outputs_equivalent(full_output, combined):
                    candidates.append(axis)
            except Exception:
                continue
        if len(candidates) == 1:
            return candidates[0]
        if len(candidates) > 1:
            raise ValueError(
                "split_axis='auto' is ambiguous: both row and column probes matched. "
                "Please set split_axis explicitly."
            )
        raise ValueError(
            "split_axis='auto' could not prove that the function is row- or column-splittable. "
            "Please set split_axis explicitly."
        )

    def _outputs_equivalent(self, left: Any, right: Any) -> bool:
        try:
            if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
                pd.testing.assert_frame_equal(left, right, check_dtype=False, check_exact=False)
                return True
            if isinstance(left, pd.Series) and isinstance(right, pd.Series):
                pd.testing.assert_series_equal(left, right, check_dtype=False, check_exact=False)
                return True
            if isinstance(left, np.ndarray) and isinstance(right, np.ndarray):
                return bool(np.allclose(left, right, equal_nan=True))
            return left == right
        except Exception:
            return False

    def _make_chunks(
        self,
        data: Any,
        chunks: Sequence[Any] | None,
        split_axis: str,
        force_n_chunks: int | None = None,
    ) -> list[_ChunkSpec]:
        if split_axis == "chunk":
            return [
                _ChunkSpec(chunk_id=i, chunk=chunk, rows=self._safe_len(chunk))
                for i, chunk in enumerate(chunks or [])
            ]
        if split_axis == "row":
            return self._make_row_chunks(data, force_n_chunks)
        if split_axis == "column":
            return self._make_column_chunks(data, force_n_chunks)
        raise ValueError(f"Unsupported split_axis: {split_axis}")

    def _make_row_chunks(self, data: pd.DataFrame, force_n_chunks: int | None = None) -> list[_ChunkSpec]:
        indices = self._split_positions(len(data), force_n_chunks)
        return [
            _ChunkSpec(chunk_id=i, chunk=data.iloc[pos].copy(), rows=len(pos), columns=list(data.columns))
            for i, pos in enumerate(indices)
            if len(pos) > 0
        ]

    def _make_column_chunks(self, data: pd.DataFrame, force_n_chunks: int | None = None) -> list[_ChunkSpec]:
        fixed_cols = self._dedupe(self.config.id_cols + self.config.required_cols)
        missing = [col for col in fixed_cols if col not in data.columns]
        if missing:
            raise KeyError(f"required/id columns not found in data: {missing}")
        split_cols = [col for col in data.columns if col not in set(fixed_cols)]
        positions = self._split_positions(len(split_cols), force_n_chunks)
        specs = []
        for i, pos in enumerate(positions):
            cols = fixed_cols + [split_cols[j] for j in pos]
            if len(cols) == 0:
                continue
            specs.append(
                _ChunkSpec(
                    chunk_id=i,
                    chunk=data.loc[:, cols].copy(),
                    rows=len(data),
                    columns=cols,
                )
            )
        return specs

    def _split_positions(self, n_items: int, force_n_chunks: int | None = None) -> list[np.ndarray]:
        if n_items < 0:
            raise ValueError("n_items must be non-negative.")
        if n_items == 0:
            return [np.array([], dtype=int)]
        if force_n_chunks is not None:
            n_chunks = min(max(1, force_n_chunks), n_items)
        elif self.config.chunk_size is not None:
            n_chunks = int(math.ceil(n_items / self.config.chunk_size))
        elif self.config.n_chunks is not None:
            n_chunks = min(self.config.n_chunks, n_items)
        else:
            n_chunks = min(self._resolve_n_jobs(self.config.n_jobs, n_items), n_items)
        return [arr for arr in np.array_split(np.arange(n_items), n_chunks) if len(arr) > 0]

    def _resolve_n_jobs(self, n_jobs: int | str | None, n_tasks: int) -> int:
        if n_tasks <= 0:
            return 1
        if n_jobs is None:
            resolved = 1
        elif isinstance(n_jobs, str):
            if n_jobs != "auto":
                raise ValueError("n_jobs as string only supports 'auto'.")
            resolved = max(1, multiprocessing.cpu_count() - 1)
        elif n_jobs == -1:
            resolved = multiprocessing.cpu_count()
        elif n_jobs <= 0:
            raise ValueError("n_jobs must be a positive integer, -1, 'auto', or None.")
        else:
            resolved = int(n_jobs)
        return min(max(1, resolved), max(1, n_tasks))

    def _execute(
        self,
        func: Callable[..., Any],
        func_args: tuple[Any, ...],
        func_kwargs: dict[str, Any],
        chunk_specs: list[_ChunkSpec],
        n_jobs: int,
    ) -> list[dict[str, Any]]:
        cfg = self.config
        collect_errors = cfg.on_error == "collect"
        if cfg.backend == "sequential" or n_jobs == 1:
            return [
                _run_chunk(
                    func,
                    spec.chunk,
                    func_args,
                    func_kwargs,
                    cfg.pass_chunk_info,
                    self._chunk_info(spec),
                    collect_errors,
                )
                for spec in chunk_specs
            ]

        from joblib import Parallel, delayed

        backend = "threading" if cfg.backend == "thread" else "loky"
        parallel_kwargs = {"n_jobs": n_jobs, "backend": backend}
        if cfg.timeout is not None:
            parallel_kwargs["timeout"] = cfg.timeout
        return Parallel(**parallel_kwargs)(
            delayed(_run_chunk)(
                func,
                spec.chunk,
                func_args,
                func_kwargs,
                cfg.pass_chunk_info,
                self._chunk_info(spec),
                collect_errors,
            )
            for spec in chunk_specs
        )

    def _combine_outputs(self, successful_results: list[dict[str, Any]], split_axis: str) -> Any:
        cfg = self.config
        outputs = [item["output"] for item in successful_results]
        if cfg.combine == "none":
            return None
        if cfg.combine == "list":
            return outputs
        if cfg.combine == "dict":
            return {item["chunk_id"]: item["output"] for item in successful_results}
        if not outputs:
            return None
        concat_axis = cfg.concat_axis
        if concat_axis is None:
            concat_axis = 1 if split_axis == "column" else 0
        return pd.concat(outputs, axis=concat_axis)

    def _chunk_info(self, spec: _ChunkSpec) -> dict[str, Any]:
        return {
            "chunk_id": spec.chunk_id,
            "rows": spec.rows,
            "columns": spec.columns,
        }

    def _safe_len(self, obj: Any) -> int | None:
        try:
            return len(obj)
        except Exception:
            return None

    def _dedupe(self, values: Sequence[str]) -> list[str]:
        seen = set()
        result = []
        for value in values:
            if value not in seen:
                seen.add(value)
                result.append(value)
        return result

run

run(data: Any = None, func: Callable[..., Any] | None = None, func_args: Sequence[Any] = (), func_kwargs: dict[str, Any] | None = None, chunks: Sequence[Any] | None = None) -> ParallelApplyResult

Split the work, run func on every chunk, and combine the outputs.

Parameters:

Name	Type	Description	Default
data	DataFrame or None	Frame to split. It must be a DataFrame unless split_axis="chunk", and also with split_axis="auto".	None
func	callable	Called as func(chunk, *func_args, **func_kwargs) for every chunk, with an extra chunk_info keyword argument when pass_chunk_info is True. It is required although the signature default is None.	None
func_args	sequence	Extra positional arguments for func.	()
func_kwargs	dict or None	Extra keyword arguments for func.	None
chunks	sequence or None	The objects to process one by one; required when split_axis="chunk". With split_axis="auto" a given chunks selects the chunk mode.	None

Returns:

Type	Description
ParallelApplyResult	The combined output, the per-chunk outputs, the collected errors and a summary.

Raises:

Type	Description
ValueError	If func is None, split_axis="chunk" has no chunks, split_axis="auto" cannot decide on a single axis, or n_jobs is invalid.
TypeError	If data is not a DataFrame where one is required, or if func and its arguments cannot be serialized for the process backend.
KeyError	If required_cols or id_cols are missing from data in a column split.

Source code in Modeling_Tool/Core/Parallel_Engine.py

def run(
    self,
    data: Any = None,
    func: Callable[..., Any] | None = None,
    func_args: Sequence[Any] = (),
    func_kwargs: dict[str, Any] | None = None,
    chunks: Sequence[Any] | None = None,
) -> ParallelApplyResult:
    """Split the work, run ``func`` on every chunk, and combine the outputs.

    Parameters
    ----------
    data : pandas.DataFrame or None, default None
        Frame to split. It must be a DataFrame unless ``split_axis="chunk"``, and also with ``split_axis="auto"``.
    func : callable
        Called as ``func(chunk, *func_args, **func_kwargs)`` for every chunk, with an extra ``chunk_info`` keyword
        argument when ``pass_chunk_info`` is True. It is required although the signature default is None.
    func_args : sequence, default ()
        Extra positional arguments for ``func``.
    func_kwargs : dict or None, default None
        Extra keyword arguments for ``func``.
    chunks : sequence or None, default None
        The objects to process one by one; required when ``split_axis="chunk"``. With ``split_axis="auto"`` a
        given ``chunks`` selects the chunk mode.

    Returns
    -------
    ParallelApplyResult
        The combined ``output``, the per-chunk outputs, the collected ``errors`` and a ``summary``.

    Raises
    ------
    ValueError
        If ``func`` is None, ``split_axis="chunk"`` has no ``chunks``, ``split_axis="auto"`` cannot decide on a
        single axis, or ``n_jobs`` is invalid.
    TypeError
        If ``data`` is not a DataFrame where one is required, or if ``func`` and its arguments cannot be serialized
        for the process backend.
    KeyError
        If ``required_cols`` or ``id_cols`` are missing from ``data`` in a column split.
    """
    if func is None:
        raise ValueError("func is required.")

    cfg = self.config
    func_kwargs = {} if func_kwargs is None else dict(func_kwargs)
    func_args = tuple(func_args)
    self._validate_runtime_inputs(data=data, chunks=chunks)
    if cfg.validate_picklable and cfg.backend == "process":
        self._validate_picklable(func, func_args, func_kwargs)

    start = time.time()
    split_axis = self._resolve_split_axis(data, func, func_args, func_kwargs, chunks)
    chunk_specs = self._make_chunks(data=data, chunks=chunks, split_axis=split_axis)
    n_jobs = self._resolve_n_jobs(cfg.n_jobs, len(chunk_specs))

    raw_results = self._execute(
        func=func,
        func_args=func_args,
        func_kwargs=func_kwargs,
        chunk_specs=chunk_specs,
        n_jobs=n_jobs,
    )
    if cfg.preserve_order:
        raw_results = sorted(raw_results, key=lambda item: item["chunk_id"])

    successful = [item for item in raw_results if item["ok"]]
    chunk_outputs = [item["output"] for item in successful]
    errors = pd.DataFrame([item["error"] for item in raw_results if not item["ok"]])
    output = self._combine_outputs(successful, split_axis)
    elapsed = time.time() - start

    return ParallelApplyResult(
        output=output,
        chunk_outputs=chunk_outputs,
        errors=errors,
        summary={
            "split_axis": split_axis,
            "backend": cfg.backend,
            "n_jobs": n_jobs,
            "n_chunks": len(chunk_specs),
            "n_success": len(successful),
            "n_error": int(len(errors)),
            "elapsed_seconds": elapsed,
        },
        config=cfg,
        split_axis_resolved=split_axis,
    )

parallel_apply

parallel_apply(data: Any = None, func: Callable[..., Any] | None = None, func_args: Sequence[Any] = (), func_kwargs: dict[str, Any] | None = None, chunks: Sequence[Any] | None = None, **config_kwargs: Any) -> Any

Run a function in parallel and return the combined output.

Parameters:

Name	Type	Description	Default
data	DataFrame or None	Frame to split, as in ParallelApplyEngine.run.	None
func	callable or None	Function applied to every chunk; required.	None
func_args	sequence	Extra positional arguments for func.	()
func_kwargs	dict or None	Extra keyword arguments for func.	None
chunks	sequence or None	Objects to process one by one when split_axis="chunk".	None
**config_kwargs	Any	Fields of ParallelApplyConfig, for example n_jobs=4 or split_axis="column".	{}

Returns:

Type	Description
object	Only the combined output (ParallelApplyResult.output); use ParallelApplyEngine to get the errors and the run summary.

Source code in Modeling_Tool/Core/Parallel_Engine.py

def parallel_apply(
    data: Any = None,
    func: Callable[..., Any] | None = None,
    func_args: Sequence[Any] = (),
    func_kwargs: dict[str, Any] | None = None,
    chunks: Sequence[Any] | None = None,
    **config_kwargs: Any,
) -> Any:
    """Run a function in parallel and return the combined output.

    Parameters
    ----------
    data : pandas.DataFrame or None, default None
        Frame to split, as in ``ParallelApplyEngine.run``.
    func : callable or None, default None
        Function applied to every chunk; required.
    func_args : sequence, default ()
        Extra positional arguments for ``func``.
    func_kwargs : dict or None, default None
        Extra keyword arguments for ``func``.
    chunks : sequence or None, default None
        Objects to process one by one when ``split_axis="chunk"``.
    **config_kwargs
        Fields of ``ParallelApplyConfig``, for example ``n_jobs=4`` or ``split_axis="column"``.

    Returns
    -------
    object
        Only the combined output (``ParallelApplyResult.output``); use ``ParallelApplyEngine`` to get the errors and the
        run summary.
    """

    config = ParallelApplyConfig(**config_kwargs)
    return ParallelApplyEngine(config).run(
        data=data,
        func=func,
        func_args=func_args,
        func_kwargs=func_kwargs,
        chunks=chunks,
    ).output

Dataset Comparison: Proc_Compare

Proc_Compare

ProcCompareConfig dataclass

Settings of a ProcCompareEngine run (the keyword arguments of proc_compare).

Parameters:

Name	Type	Description	Default
output_dir	str	Folder for the CSV files, the Excel report and the temporary files of CSV mode.	"output/proc_compare"
write_outputs	bool	Write the six result tables as CSV files into output_dir.	True
write_excel	bool	Write the ExcelMaster report.	False
left_name	str	Name of the left table, written to coverage_summary and duplicate_key_summary.	"left"
right_name	str	Name of the right table.	"right"
key_cols	list of str or None	Key columns that align the two tables. Required unless row_order_compare is True.	None
row_order_compare	bool	Align the tables by row number when there is no key.	False
compare_cols	list of str or None	Columns to compare; None compares every column that exists in both tables.	None
ignore_cols	list of str	Columns excluded from the comparison.	empty list
chunk_size	int	Rows read per chunk in CSV mode; must be positive.	200000
n_partitions	int	Number of hash partitions in CSV mode; must be positive.	16
backend	('sequential', 'thread', 'process')	How the CSV partitions are compared.	"sequential"
compare_block_size	int	Columns compared per vectorized block; lower it for very wide tables to limit memory. It does not change the results. Must be positive.	64
numeric_tol	float	Absolute numeric tolerance: two numbers match when abs(left - right) <= numeric_tol + numeric_rtol * abs(right). Must not be negative.	1e-8
numeric_rtol	float	Relative numeric tolerance, applied to abs(right). Must not be negative.	0.0
datetime_tol_seconds	float	Tolerance for datetime columns, in seconds. Must not be negative.	0.0
datetime_cols	list of str	Columns to compare as datetimes (recommended in CSV mode, where the dtype cannot be inferred from the file).	empty list
per_column_tolerance	dict	Per-column overrides of the tolerances: a number (absolute tolerance), or a dict with the keys tol, rtol and datetime_tol_seconds.	empty dict
both_null_equal	bool	Count two nulls as consistent; with False they count as a mismatch.	True
missing_values	list	Extra values (for example "" or -999) to convert to missing on both sides and in every column.	empty list
detail_mode	('top', 'full', 'none')	Which mismatched cells cell_mismatches lists. "top" keeps at most top_n cells (all of them up to that number, otherwise those with the largest abs_diff), "full" keeps every mismatched cell and "none" returns an empty frame. The other result tables are always complete.	"top"
top_n	int	Number of cells kept when detail_mode="top"; must be positive.	1000
duplicate_key_policy	('raise', 'first', 'all')	What to do with duplicated keys: "raise" raises ValueError, "first" keeps the first row of each key on both sides, and "all" keeps every row (matching duplicates multiply in the merge).	"raise"
excel_output_path	str or None	Path of the Excel report; None uses <output_dir>/Proc_Compare_Report.xlsx.	None
max_excel_rows	int	Maximum number of rows written per Excel sheet.	100000

Source code in Modeling_Tool/Core/Proc_Compare.py

@dataclass
class ProcCompareConfig:
    """Settings of a ``ProcCompareEngine`` run (the keyword arguments of ``proc_compare``).

    Parameters
    ----------
    output_dir : str, default "output/proc_compare"
        Folder for the CSV files, the Excel report and the temporary files of CSV mode.
    write_outputs : bool, default True
        Write the six result tables as CSV files into ``output_dir``.
    write_excel : bool, default False
        Write the ExcelMaster report.
    left_name : str, default "left"
        Name of the left table, written to ``coverage_summary`` and ``duplicate_key_summary``.
    right_name : str, default "right"
        Name of the right table.
    key_cols : list of str or None, default None
        Key columns that align the two tables. Required unless ``row_order_compare`` is True.
    row_order_compare : bool, default False
        Align the tables by row number when there is no key.
    compare_cols : list of str or None, default None
        Columns to compare; None compares every column that exists in both tables.
    ignore_cols : list of str, default empty list
        Columns excluded from the comparison.
    chunk_size : int, default 200000
        Rows read per chunk in CSV mode; must be positive.
    n_partitions : int, default 16
        Number of hash partitions in CSV mode; must be positive.
    backend : {"sequential", "thread", "process"}, default "sequential"
        How the CSV partitions are compared.
    compare_block_size : int, default 64
        Columns compared per vectorized block; lower it for very wide tables to limit memory. It does not change the
        results. Must be positive.
    numeric_tol : float, default 1e-8
        Absolute numeric tolerance: two numbers match when ``abs(left - right) <= numeric_tol + numeric_rtol * abs(right)``.
        Must not be negative.
    numeric_rtol : float, default 0.0
        Relative numeric tolerance, applied to ``abs(right)``. Must not be negative.
    datetime_tol_seconds : float, default 0.0
        Tolerance for datetime columns, in seconds. Must not be negative.
    datetime_cols : list of str, default empty list
        Columns to compare as datetimes (recommended in CSV mode, where the dtype cannot be inferred from the file).
    per_column_tolerance : dict, default empty dict
        Per-column overrides of the tolerances: a number (absolute tolerance), or a dict with the keys ``tol``,
        ``rtol`` and ``datetime_tol_seconds``.
    both_null_equal : bool, default True
        Count two nulls as consistent; with False they count as a mismatch.
    missing_values : list, default empty list
        Extra values (for example ``""`` or ``-999``) to convert to missing on both sides and in every column.
    detail_mode : {"top", "full", "none"}, default "top"
        Which mismatched cells ``cell_mismatches`` lists. ``"top"`` keeps at most ``top_n`` cells (all of them up to that
        number, otherwise those with the largest ``abs_diff``), ``"full"`` keeps every mismatched cell and ``"none"``
        returns an empty frame. The other result tables are always complete.
    top_n : int, default 1000
        Number of cells kept when ``detail_mode="top"``; must be positive.
    duplicate_key_policy : {"raise", "first", "all"}, default "raise"
        What to do with duplicated keys: ``"raise"`` raises ``ValueError``, ``"first"`` keeps the first row of each key
        on both sides, and ``"all"`` keeps every row (matching duplicates multiply in the merge).
    excel_output_path : str or None, default None
        Path of the Excel report; None uses ``<output_dir>/Proc_Compare_Report.xlsx``.
    max_excel_rows : int, default 100000
        Maximum number of rows written per Excel sheet.
    """

    output_dir: str = "output/proc_compare"
    write_outputs: bool = True
    write_excel: bool = False
    left_name: str = "left"
    right_name: str = "right"

    key_cols: list[str] | None = None
    row_order_compare: bool = False
    compare_cols: list[str] | None = None
    ignore_cols: list[str] = field(default_factory=list)

    chunk_size: int = 200_000
    n_partitions: int = 16
    backend: CompareBackend = "sequential"
    compare_block_size: int = 64

    numeric_tol: float = 1e-8
    numeric_rtol: float = 0.0
    datetime_tol_seconds: float = 0.0
    datetime_cols: list[str] = field(default_factory=list)
    per_column_tolerance: dict[str, Any] = field(default_factory=dict)

    both_null_equal: bool = True
    missing_values: list[Any] = field(default_factory=list)

    detail_mode: DetailMode = "top"
    top_n: int = 1000
    duplicate_key_policy: DuplicateKeyPolicy = "raise"

    excel_output_path: str | None = None
    max_excel_rows: int = 100_000

ProcCompareResult dataclass

Result of ProcCompareEngine.run.

Parameters:

Name	Type	Description	Default
coverage_summary	DataFrame	One row with the row counts of both tables, the common rows and the rows found on one side only.	required
schema_summary	DataFrame	One row per column of either table with its role, dtypes and comparison status.	required
column_summary	DataFrame	One row per compared column: counts of compared, equal, mismatching and null cells, and the mean and maximum absolute difference.	required
row_summary	DataFrame	One row per key: row_status (both, left_only or right_only) and the mismatching columns.	required
cell_mismatches	DataFrame	One row per mismatching cell (limited by detail_mode and top_n).	required
duplicate_key_summary	DataFrame	One row per duplicated key and side; empty when there are no duplicates.	required
output_paths	dict	Table name to CSV path; empty unless write_outputs is True.	empty dict
report_path	str or None	Path of the Excel report, or None when it was not written.	None

Source code in Modeling_Tool/Core/Proc_Compare.py

@dataclass
class ProcCompareResult:
    """Result of ``ProcCompareEngine.run``.

    Parameters
    ----------
    coverage_summary : pandas.DataFrame
        One row with the row counts of both tables, the common rows and the rows found on one side only.
    schema_summary : pandas.DataFrame
        One row per column of either table with its role, dtypes and comparison status.
    column_summary : pandas.DataFrame
        One row per compared column: counts of compared, equal, mismatching and null cells, and the mean and maximum
        absolute difference.
    row_summary : pandas.DataFrame
        One row per key: ``row_status`` (``both``, ``left_only`` or ``right_only``) and the mismatching columns.
    cell_mismatches : pandas.DataFrame
        One row per mismatching cell (limited by ``detail_mode`` and ``top_n``).
    duplicate_key_summary : pandas.DataFrame
        One row per duplicated key and side; empty when there are no duplicates.
    output_paths : dict, default empty dict
        Table name to CSV path; empty unless ``write_outputs`` is True.
    report_path : str or None, default None
        Path of the Excel report, or None when it was not written.
    """

    coverage_summary: pd.DataFrame
    schema_summary: pd.DataFrame
    column_summary: pd.DataFrame
    row_summary: pd.DataFrame
    cell_mismatches: pd.DataFrame
    duplicate_key_summary: pd.DataFrame
    output_paths: dict[str, str] = field(default_factory=dict)
    report_path: str | None = None

ProcCompareEngine

SAS proc_compare-like consistency checker for DataFrames and CSV files.

Parameters:

Name	Type	Description	Default
config	ProcCompareConfig or None	Settings; None uses ProcCompareConfig().	None

Raises:

Type	Description
ValueError	If a setting is invalid: backend, detail_mode or duplicate_key_policy outside their allowed values, a non-positive chunk_size, n_partitions, compare_block_size or top_n, or a negative tolerance.

Source code in Modeling_Tool/Core/Proc_Compare.py

class ProcCompareEngine:
    """SAS proc_compare-like consistency checker for DataFrames and CSV files.

    Parameters
    ----------
    config : ProcCompareConfig or None, default None
        Settings; None uses ``ProcCompareConfig()``.

    Raises
    ------
    ValueError
        If a setting is invalid: ``backend``, ``detail_mode`` or ``duplicate_key_policy`` outside their allowed values,
        a non-positive ``chunk_size``, ``n_partitions``, ``compare_block_size`` or ``top_n``, or a negative tolerance.
    """

    _VALID_BACKENDS = {"sequential", "thread", "process"}
    _VALID_DETAIL_MODES = {"top", "full", "none"}
    _VALID_DUPLICATE_POLICIES = {"raise", "first", "all"}

    def __init__(self, config: ProcCompareConfig | None = None):
        self.config = config or ProcCompareConfig()
        self._validate_config()

    def run(self, left: pd.DataFrame | str | Path, right: pd.DataFrame | str | Path) -> ProcCompareResult:
        """Compare two tables.

        When either input is a CSV path the comparison runs in CSV mode (chunked reading and hash partitions);
        otherwise both frames are compared in memory.

        Parameters
        ----------
        left : pandas.DataFrame, str or pathlib.Path
            The left table, or the path of a CSV file.
        right : pandas.DataFrame, str or pathlib.Path
            The right table, or the path of a CSV file.

        Returns
        -------
        ProcCompareResult
            The six result tables, plus the CSV paths and the Excel report path when they were written.

        Raises
        ------
        ValueError
            If neither ``key_cols`` nor ``row_order_compare`` is set, if a key column is missing from a table, or if a key
            is duplicated while ``duplicate_key_policy="raise"``.
        """
        key_cols = self._resolve_key_cols()
        if self._is_csv_like(left) or self._is_csv_like(right):
            result = self._run_csv(left, right, key_cols)
        else:
            left_df = self._prepare_dataframe(left, side=self.config.left_name, key_cols=key_cols)
            right_df = self._prepare_dataframe(right, side=self.config.right_name, key_cols=key_cols)
            schema = self._build_schema_summary(left_df, right_df, key_cols)
            result = self._compare_frames(left_df, right_df, key_cols, schema_summary=schema)

        output_paths: dict[str, str] = {}
        if self.config.write_outputs:
            output_paths = self._write_outputs(result)
            result.output_paths.update(output_paths)
        if self.config.write_excel:
            result.report_path = self._write_excel(result)
        return result

    def _validate_config(self) -> None:
        cfg = self.config
        if cfg.backend not in self._VALID_BACKENDS:
            raise ValueError(f"backend must be one of {sorted(self._VALID_BACKENDS)}.")
        if cfg.detail_mode not in self._VALID_DETAIL_MODES:
            raise ValueError(f"detail_mode must be one of {sorted(self._VALID_DETAIL_MODES)}.")
        if cfg.duplicate_key_policy not in self._VALID_DUPLICATE_POLICIES:
            raise ValueError(
                f"duplicate_key_policy must be one of {sorted(self._VALID_DUPLICATE_POLICIES)}."
            )
        if cfg.chunk_size <= 0:
            raise ValueError("chunk_size must be a positive integer.")
        if cfg.n_partitions <= 0:
            raise ValueError("n_partitions must be a positive integer.")
        if cfg.compare_block_size <= 0:
            raise ValueError("compare_block_size must be a positive integer.")
        if cfg.numeric_tol < 0 or cfg.numeric_rtol < 0 or cfg.datetime_tol_seconds < 0:
            raise ValueError("tolerance values must be non-negative.")
        if cfg.top_n <= 0:
            raise ValueError("top_n must be a positive integer.")

    def _resolve_key_cols(self) -> list[str]:
        key_cols = list(self.config.key_cols or [])
        if key_cols:
            return key_cols
        if self.config.row_order_compare:
            return [_ROW_ID_COL]
        raise ValueError("Provide key_cols or set row_order_compare=True.")

    @staticmethod
    def _is_csv_like(value: Any) -> bool:
        return isinstance(value, (str, Path))

    def _prepare_dataframe(
        self,
        data: pd.DataFrame | str | Path,
        side: str,
        key_cols: list[str],
    ) -> pd.DataFrame:
        if isinstance(data, pd.DataFrame):
            df = data.copy()
        else:
            df = pd.read_csv(data)
        if key_cols == [_ROW_ID_COL] and _ROW_ID_COL not in df.columns:
            df = df.copy()
            df[_ROW_ID_COL] = np.arange(len(df), dtype=np.int64)
        self._validate_columns(df, key_cols, side)
        return self._normalize_missing(df)

    def _validate_columns(self, df: pd.DataFrame, key_cols: list[str], side: str) -> None:
        missing = [col for col in key_cols if col not in df.columns]
        if missing:
            raise ValueError(f"{side} dataset is missing key columns: {missing}")

    def _normalize_missing(self, df: pd.DataFrame) -> pd.DataFrame:
        if not self.config.missing_values:
            return df
        return df.replace(self.config.missing_values, pd.NA)

    def _handle_duplicates(
        self,
        df: pd.DataFrame,
        key_cols: list[str],
        side: str,
    ) -> tuple[pd.DataFrame, pd.DataFrame]:
        if key_cols == [_ROW_ID_COL]:
            return df, self._empty_duplicate_summary()
        dup_mask = df.duplicated(subset=key_cols, keep=False)
        if not dup_mask.any():
            return df, self._empty_duplicate_summary()

        dup = df.loc[dup_mask, key_cols].copy()
        grouped = dup.value_counts(key_cols).reset_index(name="duplicate_rows")
        grouped.insert(0, "side", side)

        if self.config.duplicate_key_policy == "raise":
            examples = grouped.head(5).to_dict("records")
            raise ValueError(f"{side} dataset has duplicate keys; examples={examples}")
        if self.config.duplicate_key_policy == "first":
            df = df.drop_duplicates(subset=key_cols, keep="first")
        return df, grouped

    @staticmethod
    def _empty_duplicate_summary() -> pd.DataFrame:
        return pd.DataFrame(columns=["side", "duplicate_rows"])

    def _build_schema_summary(
        self,
        left_df: pd.DataFrame,
        right_df: pd.DataFrame,
        key_cols: list[str],
    ) -> pd.DataFrame:
        cols = sorted((set(left_df.columns) | set(right_df.columns)) - {_ROW_ID_COL})
        rows = []
        ignore = set(self.config.ignore_cols)
        requested_compare_set = set(
            self._resolve_compare_cols(left_df, right_df, key_cols, allow_missing=True)
        )
        for col in cols:
            in_left = col in left_df.columns
            in_right = col in right_df.columns
            requested_for_compare = col in requested_compare_set
            eligible_for_compare = bool(
                requested_for_compare
                and in_left
                and in_right
                and col not in key_cols
                and col not in ignore
            )
            if col in key_cols:
                role = "key"
            elif not (in_left and in_right):
                role = "schema_only"
            elif col in ignore:
                role = "ignored"
            elif eligible_for_compare:
                role = "compare"
            else:
                role = "not_compared"
            if in_left and in_right:
                status = "both"
            elif in_left:
                status = "left_only_column"
            else:
                status = "right_only_column"
            rows.append(
                {
                    "column": col,
                    "role": role,
                    "requested_for_compare": bool(requested_for_compare),
                    "eligible_for_compare": bool(eligible_for_compare),
                    "in_left": bool(in_left),
                    "in_right": bool(in_right),
                    "dtype_left": str(left_df[col].dtype) if in_left else "",
                    "dtype_right": str(right_df[col].dtype) if in_right else "",
                    "status": status,
                    "dtype_equal": bool(in_left and in_right and str(left_df[col].dtype) == str(right_df[col].dtype)),
                }
            )
        return pd.DataFrame(rows)

    def _resolve_compare_cols(
        self,
        left_df: pd.DataFrame,
        right_df: pd.DataFrame,
        key_cols: list[str],
        allow_missing: bool = False,
    ) -> list[str]:
        ignore = set(self.config.ignore_cols) | set(key_cols) | {_ROW_ID_COL}
        if self.config.compare_cols is not None:
            candidates = [col for col in self.config.compare_cols if col not in ignore]
        else:
            candidates = sorted((set(left_df.columns) | set(right_df.columns)) - ignore)
        if allow_missing:
            return candidates
        return [col for col in candidates if col in left_df.columns and col in right_df.columns]

    def _compare_frames(
        self,
        left_df: pd.DataFrame,
        right_df: pd.DataFrame,
        key_cols: list[str],
        schema_summary: pd.DataFrame | None = None,
        keep_internal_stats: bool = False,
    ) -> ProcCompareResult:
        left_df, left_dups = self._handle_duplicates(left_df, key_cols, self.config.left_name)
        right_df, right_dups = self._handle_duplicates(right_df, key_cols, self.config.right_name)
        duplicate_key_summary = pd.concat([left_dups, right_dups], ignore_index=True)
        schema_summary = schema_summary if schema_summary is not None else self._build_schema_summary(left_df, right_df, key_cols)
        compare_cols = self._resolve_compare_cols(left_df, right_df, key_cols)

        merged = left_df.merge(
            right_df,
            on=key_cols,
            how="outer",
            suffixes=("_left", "_right"),
            indicator=True,
        )
        common_mask = merged["_merge"].eq("both")
        coverage_summary = pd.DataFrame(
            [
                {
                    "left_name": self.config.left_name,
                    "right_name": self.config.right_name,
                    "n_left_rows": int(len(left_df)),
                    "n_right_rows": int(len(right_df)),
                    "n_common_rows": int(common_mask.sum()),
                    "n_left_only_rows": int(merged["_merge"].eq("left_only").sum()),
                    "n_right_only_rows": int(merged["_merge"].eq("right_only").sum()),
                    "n_compare_columns": int(len(compare_cols)),
                }
            ]
        )

        row_summary = merged[key_cols + ["_merge"]].copy()
        row_summary = row_summary.rename(columns={"_merge": "row_status"})
        row_summary["n_cell_mismatch"] = 0
        row_summary["mismatch_columns"] = ""

        column_rows = []
        mismatch_frames = []
        mismatch_cols_by_row: dict[int, list[str]] = {}

        common_values = common_mask.to_numpy(dtype=bool)
        n_compared = int(common_values.sum())
        block_size = int(self.config.compare_block_size)
        for start in range(0, len(compare_cols), block_size):
            block_cols = compare_cols[start : start + block_size]
            comp = self._compare_column_block(merged, block_cols, common_values)
            mismatch_matrix = comp["mismatch"]
            one_null_matrix = comp["one_null"]
            both_null_matrix = comp["both_null"]
            diff_matrix = comp["diff"]
            abs_diff_matrix = comp["abs_diff"]
            left_values = comp["left_values"]
            right_values = comp["right_values"]

            for col_idx, col in enumerate(block_cols):
                mismatch_positions = np.flatnonzero(mismatch_matrix[:, col_idx])
                n_mismatch = int(len(mismatch_positions))
                for idx in mismatch_positions:
                    mismatch_cols_by_row.setdefault(int(idx), []).append(col)
                if n_mismatch and self.config.detail_mode != "none":
                    detail = merged.iloc[mismatch_positions][key_cols].copy()
                    detail["column"] = col
                    detail["left_value"] = left_values[mismatch_positions, col_idx]
                    detail["right_value"] = right_values[mismatch_positions, col_idx]
                    detail["diff"] = diff_matrix[mismatch_positions, col_idx]
                    detail["abs_diff"] = abs_diff_matrix[mismatch_positions, col_idx]
                    mismatch_frames.append(detail)

                diff_values = diff_matrix[:, col_idx]
                abs_diff_values = abs_diff_matrix[:, col_idx]
                diff_valid = np.isfinite(diff_values)
                diff_sum = float(np.nansum(diff_values)) if diff_valid.any() else 0.0
                diff_count = int(diff_valid.sum())
                column_rows.append(
                    {
                        "column": col,
                        "n_compared": n_compared,
                        "n_equal": n_compared - n_mismatch,
                        "n_mismatch": n_mismatch,
                        "pct_mismatch": round(n_mismatch / n_compared * 100, 6) if n_compared else 0.0,
                        "n_one_side_null": int(one_null_matrix[:, col_idx].sum()),
                        "n_both_null": int(both_null_matrix[:, col_idx].sum()),
                        "mean_diff": diff_sum / diff_count if diff_count else np.nan,
                        "max_abs_diff": float(np.nanmax(abs_diff_values)) if np.isfinite(abs_diff_values).any() else np.nan,
                        "_diff_sum": diff_sum,
                        "_diff_count": diff_count,
                    }
                )

        for idx, cols in mismatch_cols_by_row.items():
            row_summary.loc[idx, "n_cell_mismatch"] = len(cols)
            row_summary.loc[idx, "mismatch_columns"] = ", ".join(cols)

        column_summary = pd.DataFrame(
            column_rows,
            columns=[
                "column",
                "n_compared",
                "n_equal",
                "n_mismatch",
                "pct_mismatch",
                "n_one_side_null",
                "n_both_null",
                "mean_diff",
                "max_abs_diff",
                "_diff_sum",
                "_diff_count",
            ],
        )
        column_summary = column_summary.sort_values(["n_mismatch", "column"], ascending=[False, True])

        cell_mismatches = (
            pd.concat(mismatch_frames, ignore_index=True)
            if mismatch_frames and self.config.detail_mode != "none"
            else self._empty_cell_mismatches(key_cols)
        )
        cell_mismatches = self._limit_mismatch_details(cell_mismatches)

        return ProcCompareResult(
            coverage_summary=coverage_summary,
            schema_summary=schema_summary,
            column_summary=(
                column_summary
                if keep_internal_stats
                else self._strip_internal_columns(column_summary)
            ),
            row_summary=row_summary,
            cell_mismatches=cell_mismatches,
            duplicate_key_summary=duplicate_key_summary,
        )

    def _compare_column_block(
        self,
        merged: pd.DataFrame,
        columns: list[str],
        common_mask: np.ndarray,
    ) -> dict[str, np.ndarray]:
        """Compare a bounded field block with vectorized type-specific kernels."""
        n_rows = len(merged)
        n_cols = len(columns)
        left_series = [self._normalize_series_missing(merged[f"{col}_left"]) for col in columns]
        right_series = [self._normalize_series_missing(merged[f"{col}_right"]) for col in columns]
        left_values = np.column_stack([series.to_numpy(dtype=object) for series in left_series])
        right_values = np.column_stack([series.to_numpy(dtype=object) for series in right_series])
        mismatch = np.zeros((n_rows, n_cols), dtype=bool)
        one_null = np.zeros((n_rows, n_cols), dtype=bool)
        both_null = np.zeros((n_rows, n_cols), dtype=bool)
        diff = np.full((n_rows, n_cols), np.nan, dtype=float)
        abs_diff = np.full((n_rows, n_cols), np.nan, dtype=float)

        datetime_idx = [
            idx
            for idx, col in enumerate(columns)
            if self._should_compare_datetime(col, left_series[idx], right_series[idx])
        ]
        datetime_set = set(datetime_idx)
        numeric_idx = [
            idx
            for idx in range(n_cols)
            if idx not in datetime_set
            and self._should_compare_numeric(left_series[idx], right_series[idx])
        ]
        numeric_set = set(numeric_idx)
        string_idx = [
            idx
            for idx in range(n_cols)
            if idx not in datetime_set and idx not in numeric_set
        ]
        common = common_mask[:, None]

        if datetime_idx:
            left_dt = np.column_stack(
                [self._datetime_seconds(left_series[idx]) for idx in datetime_idx]
            )
            right_dt = np.column_stack(
                [self._datetime_seconds(right_series[idx]) for idx in datetime_idx]
            )
            block_diff = left_dt - right_dt
            block_abs = np.abs(block_diff)
            left_null = ~np.isfinite(left_dt)
            right_null = ~np.isfinite(right_dt)
            tolerances = np.asarray(
                [self._column_datetime_tol(columns[idx]) for idx in datetime_idx],
                dtype=float,
            )
            block_one = (left_null ^ right_null) & common
            block_both = left_null & right_null & common
            block_mismatch = (block_abs > tolerances[None, :]) | block_one
            if not self.config.both_null_equal:
                block_mismatch |= block_both
            block_mismatch &= common
            diff[:, datetime_idx] = block_diff
            abs_diff[:, datetime_idx] = block_abs
            one_null[:, datetime_idx] = block_one
            both_null[:, datetime_idx] = block_both
            mismatch[:, datetime_idx] = block_mismatch

        if numeric_idx:
            left_num = np.column_stack(
                [pd.to_numeric(left_series[idx], errors="coerce").to_numpy(dtype=float, na_value=np.nan) for idx in numeric_idx]
            )
            right_num = np.column_stack(
                [pd.to_numeric(right_series[idx], errors="coerce").to_numpy(dtype=float, na_value=np.nan) for idx in numeric_idx]
            )
            block_diff = left_num - right_num
            block_abs = np.abs(block_diff)
            left_null = ~np.isfinite(left_num)
            right_null = ~np.isfinite(right_num)
            tolerances = np.asarray(
                [self._column_numeric_tol(columns[idx]) for idx in numeric_idx],
                dtype=float,
            )
            limit = tolerances[:, 0][None, :] + tolerances[:, 1][None, :] * np.abs(right_num)
            block_one = (left_null ^ right_null) & common
            block_both = left_null & right_null & common
            block_mismatch = (block_abs > limit) | block_one
            if not self.config.both_null_equal:
                block_mismatch |= block_both
            block_mismatch &= common
            diff[:, numeric_idx] = block_diff
            abs_diff[:, numeric_idx] = block_abs
            one_null[:, numeric_idx] = block_one
            both_null[:, numeric_idx] = block_both
            mismatch[:, numeric_idx] = block_mismatch

        if string_idx:
            left_null = np.column_stack([left_series[idx].isna().to_numpy() for idx in string_idx])
            right_null = np.column_stack([right_series[idx].isna().to_numpy() for idx in string_idx])
            left_text = np.column_stack(
                [left_series[idx].astype("string").fillna("").astype(str).to_numpy() for idx in string_idx]
            )
            right_text = np.column_stack(
                [right_series[idx].astype("string").fillna("").astype(str).to_numpy() for idx in string_idx]
            )
            block_one = (left_null ^ right_null) & common
            block_both = left_null & right_null & common
            block_mismatch = (left_text != right_text) | block_one
            if self.config.both_null_equal:
                block_mismatch &= ~block_both
            else:
                block_mismatch |= block_both
            block_mismatch &= common
            one_null[:, string_idx] = block_one
            both_null[:, string_idx] = block_both
            mismatch[:, string_idx] = block_mismatch

        return {
            "mismatch": mismatch,
            "one_null": one_null,
            "both_null": both_null,
            "diff": diff,
            "abs_diff": abs_diff,
            "left_values": left_values,
            "right_values": right_values,
        }

    @staticmethod
    def _datetime_seconds(series: pd.Series) -> np.ndarray:
        values = ProcCompareEngine._to_datetime_series(series)
        missing = values.isna().to_numpy()
        result = values.astype("int64").to_numpy(dtype=float) / 1_000_000_000.0
        result[missing] = np.nan
        return result

    @staticmethod
    def _to_datetime_series(series: pd.Series) -> pd.Series:
        try:
            return pd.to_datetime(series, errors="coerce", format="mixed")
        except (TypeError, ValueError):
            return pd.to_datetime(series, errors="coerce")

    def _compare_series(
        self,
        col: str,
        left: pd.Series,
        right: pd.Series,
        common_mask: pd.Series,
    ) -> dict[str, pd.Series]:
        left = self._normalize_series_missing(left)
        right = self._normalize_series_missing(right)
        both_null = left.isna() & right.isna() & common_mask
        one_null = (left.isna() ^ right.isna()) & common_mask

        diff = pd.Series(np.nan, index=left.index, dtype="float64")
        abs_diff = pd.Series(np.nan, index=left.index, dtype="float64")

        if self._should_compare_datetime(col, left, right):
            left_dt = self._to_datetime_series(left)
            right_dt = self._to_datetime_series(right)
            diff = (left_dt - right_dt).dt.total_seconds()
            abs_diff = diff.abs()
            tol = self._column_datetime_tol(col)
            value_mismatch = abs_diff > tol
            one_null = (left_dt.isna() ^ right_dt.isna()) & common_mask
            both_null = left_dt.isna() & right_dt.isna() & common_mask
        elif self._should_compare_numeric(left, right):
            left_num = pd.to_numeric(left, errors="coerce")
            right_num = pd.to_numeric(right, errors="coerce")
            diff = left_num - right_num
            abs_diff = diff.abs()
            tol, rtol = self._column_numeric_tol(col)
            value_mismatch = abs_diff > (tol + rtol * right_num.abs())
            one_null = (left_num.isna() ^ right_num.isna()) & common_mask
            both_null = left_num.isna() & right_num.isna() & common_mask
        else:
            left_str = left.astype("string")
            right_str = right.astype("string")
            value_mismatch = left_str.ne(right_str)

        if self.config.both_null_equal:
            value_mismatch = value_mismatch & ~both_null
            mismatch = (value_mismatch | one_null) & common_mask
        else:
            mismatch = (value_mismatch | one_null | both_null) & common_mask
        return {
            "mismatch": mismatch.fillna(False),
            "one_null": one_null.fillna(False),
            "both_null": both_null.fillna(False),
            "diff": diff,
            "abs_diff": abs_diff,
        }

    def _normalize_series_missing(self, series: pd.Series) -> pd.Series:
        if not self.config.missing_values:
            return series
        return series.replace(self.config.missing_values, pd.NA)

    @staticmethod
    def _should_compare_numeric(left: pd.Series, right: pd.Series) -> bool:
        if pd.api.types.is_numeric_dtype(left) or pd.api.types.is_numeric_dtype(right):
            return True
        left_num = pd.to_numeric(left.dropna(), errors="coerce")
        right_num = pd.to_numeric(right.dropna(), errors="coerce")
        if len(left_num) == 0 and len(right_num) == 0:
            return False
        return bool(left_num.notna().all() and right_num.notna().all())

    def _should_compare_datetime(self, col: str, left: pd.Series, right: pd.Series) -> bool:
        if pd.api.types.is_datetime64_any_dtype(left) or pd.api.types.is_datetime64_any_dtype(right):
            return True
        if col in self.config.datetime_cols:
            return True

        override = self.config.per_column_tolerance.get(col)
        if isinstance(override, dict) and "datetime_tol_seconds" in override:
            return True
        if self.config.datetime_tol_seconds <= 0:
            return False
        if self._should_compare_numeric(left, right):
            return False

        values = pd.concat([left.dropna(), right.dropna()], ignore_index=True)
        if values.empty:
            return False
        text = values.astype("string").str.strip()
        if not text.str.match(r"^\d{4}[-/]\d{1,2}[-/]\d{1,2}(?:[ T].*)?$", na=False).all():
            return False
        parsed = pd.to_datetime(values, errors="coerce")
        return bool(parsed.notna().all())

    def _column_numeric_tol(self, col: str) -> tuple[float, float]:
        override = self.config.per_column_tolerance.get(col)
        if override is None:
            return float(self.config.numeric_tol), float(self.config.numeric_rtol)
        if isinstance(override, dict):
            return float(override.get("tol", self.config.numeric_tol)), float(
                override.get("rtol", self.config.numeric_rtol)
            )
        return float(override), float(self.config.numeric_rtol)

    def _column_datetime_tol(self, col: str) -> float:
        override = self.config.per_column_tolerance.get(col)
        if isinstance(override, dict):
            return float(override.get("datetime_tol_seconds", self.config.datetime_tol_seconds))
        if override is not None:
            return float(override)
        return float(self.config.datetime_tol_seconds)

    def _empty_cell_mismatches(self, key_cols: list[str]) -> pd.DataFrame:
        return pd.DataFrame(columns=key_cols + ["column", "left_value", "right_value", "diff", "abs_diff"])

    def _limit_mismatch_details(self, df: pd.DataFrame) -> pd.DataFrame:
        if self.config.detail_mode == "none":
            return df.iloc[0:0].copy()
        if self.config.detail_mode == "full" or len(df) <= self.config.top_n:
            return df
        if "abs_diff" in df.columns:
            ranked = df.copy()
            ranked["_rank_abs_diff"] = pd.to_numeric(ranked["abs_diff"], errors="coerce").fillna(-math.inf)
            ranked = ranked.sort_values("_rank_abs_diff", ascending=False).drop(columns=["_rank_abs_diff"])
            return ranked.head(self.config.top_n).reset_index(drop=True)
        return df.head(self.config.top_n).reset_index(drop=True)

    @staticmethod
    def _strip_internal_columns(df: pd.DataFrame) -> pd.DataFrame:
        return df[[col for col in df.columns if not col.startswith("_")]].copy()

    def _run_csv(
        self,
        left: pd.DataFrame | str | Path,
        right: pd.DataFrame | str | Path,
        key_cols: list[str],
    ) -> ProcCompareResult:
        left_df_for_schema = pd.read_csv(left, nrows=1000) if self._is_csv_like(left) else left.copy()
        right_df_for_schema = pd.read_csv(right, nrows=1000) if self._is_csv_like(right) else right.copy()
        if key_cols == [_ROW_ID_COL]:
            if _ROW_ID_COL not in left_df_for_schema.columns:
                left_df_for_schema[_ROW_ID_COL] = np.arange(len(left_df_for_schema), dtype=np.int64)
            if _ROW_ID_COL not in right_df_for_schema.columns:
                right_df_for_schema[_ROW_ID_COL] = np.arange(len(right_df_for_schema), dtype=np.int64)
        schema = self._build_schema_summary(left_df_for_schema, right_df_for_schema, key_cols)

        output_dir = Path(self.config.output_dir)
        temp_parent = output_dir if self.config.write_outputs or self.config.write_excel else None
        if temp_parent is not None:
            temp_parent.mkdir(parents=True, exist_ok=True)
        temp_dir = Path(tempfile.mkdtemp(prefix="proc_compare_", dir=str(temp_parent) if temp_parent else None))
        try:
            left_parts = temp_dir / "left"
            right_parts = temp_dir / "right"
            left_parts.mkdir()
            right_parts.mkdir()
            self._partition_input(left, left_parts, key_cols)
            self._partition_input(right, right_parts, key_cols)

            partition_ids = list(range(self.config.n_partitions))
            if self.config.backend == "sequential":
                partials = [
                    self._compare_partition(
                        part_id,
                        left_parts,
                        right_parts,
                        list(left_df_for_schema.columns),
                        list(right_df_for_schema.columns),
                        key_cols,
                        schema,
                    )
                    for part_id in partition_ids
                ]
            else:
                from .Parallel_Engine import ParallelApplyConfig, ParallelApplyEngine

                engine = ParallelApplyEngine(
                    ParallelApplyConfig(
                        split_axis="chunk",
                        backend=self.config.backend,
                        combine="list",
                        validate_picklable=False,
                    )
                )
                partials = engine.run(
                    chunks=partition_ids,
                    func=self._compare_partition,
                    func_args=(
                        left_parts,
                        right_parts,
                        list(left_df_for_schema.columns),
                        list(right_df_for_schema.columns),
                        key_cols,
                        schema,
                    ),
                ).output
            return self._aggregate_partials(partials, schema)
        finally:
            shutil.rmtree(temp_dir, ignore_errors=True)

    def _compare_partition(
        self,
        part_id: int,
        left_parts: Path,
        right_parts: Path,
        left_columns: list[str],
        right_columns: list[str],
        key_cols: list[str],
        schema_summary: pd.DataFrame,
    ) -> ProcCompareResult:
        left_part = left_parts / f"part_{part_id}.csv"
        right_part = right_parts / f"part_{part_id}.csv"
        left_part_df = pd.read_csv(left_part) if left_part.exists() else pd.DataFrame(columns=left_columns)
        right_part_df = pd.read_csv(right_part) if right_part.exists() else pd.DataFrame(columns=right_columns)
        left_part_df = self._normalize_missing(left_part_df)
        right_part_df = self._normalize_missing(right_part_df)
        return self._compare_frames(
            left_part_df,
            right_part_df,
            key_cols,
            schema_summary=schema_summary,
            keep_internal_stats=True,
        )

    def _partition_input(self, data: pd.DataFrame | str | Path, out_dir: Path, key_cols: list[str]) -> None:
        if isinstance(data, pd.DataFrame):
            iterator = [data.copy()]
        else:
            iterator = pd.read_csv(data, chunksize=self.config.chunk_size)

        row_offset = 0
        for chunk in iterator:
            if key_cols == [_ROW_ID_COL] and _ROW_ID_COL not in chunk.columns:
                chunk = chunk.copy()
                chunk[_ROW_ID_COL] = np.arange(row_offset, row_offset + len(chunk), dtype=np.int64)
            self._validate_columns(chunk, key_cols, str(out_dir.name))
            hashes = pd.util.hash_pandas_object(chunk[key_cols], index=False)
            partitions = (hashes % self.config.n_partitions).astype(int)
            for part_id, part_df in chunk.groupby(partitions, sort=False):
                path = out_dir / f"part_{int(part_id)}.csv"
                part_df.to_csv(path, mode="a", header=not path.exists(), index=False)
            row_offset += len(chunk)

    def _aggregate_partials(
        self,
        partials: list[ProcCompareResult],
        schema_summary: pd.DataFrame,
    ) -> ProcCompareResult:
        coverage = pd.concat([p.coverage_summary for p in partials], ignore_index=True)
        coverage_summary = pd.DataFrame(
            [
                {
                    "left_name": self.config.left_name,
                    "right_name": self.config.right_name,
                    "n_left_rows": int(coverage["n_left_rows"].sum()),
                    "n_right_rows": int(coverage["n_right_rows"].sum()),
                    "n_common_rows": int(coverage["n_common_rows"].sum()),
                    "n_left_only_rows": int(coverage["n_left_only_rows"].sum()),
                    "n_right_only_rows": int(coverage["n_right_only_rows"].sum()),
                    "n_compare_columns": int(coverage["n_compare_columns"].max()) if len(coverage) else 0,
                }
            ]
        )

        cols = pd.concat(
            [p.column_summary.assign(_partial=idx) for idx, p in enumerate(partials)],
            ignore_index=True,
        )
        if len(cols):
            grouped = cols.groupby("column", dropna=False).agg(
                n_compared=("n_compared", "sum"),
                n_equal=("n_equal", "sum"),
                n_mismatch=("n_mismatch", "sum"),
                n_one_side_null=("n_one_side_null", "sum"),
                n_both_null=("n_both_null", "sum"),
                _diff_sum=("_diff_sum", "sum"),
                _diff_count=("_diff_count", "sum"),
                max_abs_diff=("max_abs_diff", "max"),
            ).reset_index()
            grouped["pct_mismatch"] = np.where(
                grouped["n_compared"] > 0,
                grouped["n_mismatch"] / grouped["n_compared"] * 100,
                0.0,
            )
            grouped["mean_diff"] = np.where(
                grouped["_diff_count"] > 0,
                grouped["_diff_sum"] / grouped["_diff_count"],
                np.nan,
            )
            column_summary = grouped[
                [
                    "column",
                    "n_compared",
                    "n_equal",
                    "n_mismatch",
                    "pct_mismatch",
                    "n_one_side_null",
                    "n_both_null",
                    "mean_diff",
                    "max_abs_diff",
                ]
            ].sort_values(["n_mismatch", "column"], ascending=[False, True])
        else:
            column_summary = pd.DataFrame()

        row_summary = pd.concat([p.row_summary for p in partials], ignore_index=True)
        mismatch_tables = [p.cell_mismatches for p in partials if not p.cell_mismatches.empty]
        cell_mismatches = (
            pd.concat(mismatch_tables, ignore_index=True)
            if mismatch_tables
            else self._empty_cell_mismatches(partials[0].cell_mismatches.columns[:-5].tolist() if partials else [])
        )
        cell_mismatches = self._limit_mismatch_details(cell_mismatches)
        duplicate_key_summary = pd.concat([p.duplicate_key_summary for p in partials], ignore_index=True)

        return ProcCompareResult(
            coverage_summary=coverage_summary,
            schema_summary=schema_summary,
            column_summary=column_summary,
            row_summary=row_summary,
            cell_mismatches=cell_mismatches,
            duplicate_key_summary=duplicate_key_summary,
        )

    def _write_outputs(self, result: ProcCompareResult) -> dict[str, str]:
        output_dir = Path(self.config.output_dir)
        output_dir.mkdir(parents=True, exist_ok=True)
        tables = {
            "coverage_summary": result.coverage_summary,
            "schema_summary": result.schema_summary,
            "column_summary": result.column_summary,
            "row_summary": result.row_summary,
            "cell_mismatches": result.cell_mismatches,
            "duplicate_key_summary": result.duplicate_key_summary,
        }
        paths: dict[str, str] = {}
        for name, df in tables.items():
            path = output_dir / f"{name}.csv"
            df.to_csv(path, index=False)
            paths[name] = str(path)
        return paths

    def _write_excel(self, result: ProcCompareResult) -> str:
        from ExcelMaster.ExcelMaster import ExcelMaster

        excel_path = (
            Path(self.config.excel_output_path)
            if self.config.excel_output_path
            else Path(self.config.output_dir) / "Proc_Compare_Report.xlsx"
        )
        excel_path.parent.mkdir(parents=True, exist_ok=True)
        em = ExcelMaster(str(excel_path), verbose=False)
        tables = {
            "Coverage": result.coverage_summary,
            "Schema": result.schema_summary,
            "Column_Summary": result.column_summary,
            "Row_Summary": result.row_summary,
            "Cell_Mismatches": result.cell_mismatches,
            "Duplicate_Keys": result.duplicate_key_summary,
        }
        for sheet_name, df in tables.items():
            ws = em.add_worksheet(sheet_name[:31], zoom_perc=90)
            out = df.head(self.config.max_excel_rows).copy()
            em.write_dataframe(ws, df=out, title=sheet_name, index=False)
        em.close_workbook()
        return str(excel_path)

run

run(left: DataFrame | str | Path, right: DataFrame | str | Path) -> ProcCompareResult

Compare two tables.

When either input is a CSV path the comparison runs in CSV mode (chunked reading and hash partitions); otherwise both frames are compared in memory.

Parameters:

Name	Type	Description	Default
left	(DataFrame, str or Path)	The left table, or the path of a CSV file.	required
right	(DataFrame, str or Path)	The right table, or the path of a CSV file.	required

Returns:

Type	Description
ProcCompareResult	The six result tables, plus the CSV paths and the Excel report path when they were written.

Raises:

Type	Description
ValueError	If neither key_cols nor row_order_compare is set, if a key column is missing from a table, or if a key is duplicated while duplicate_key_policy="raise".

Source code in Modeling_Tool/Core/Proc_Compare.py

def run(self, left: pd.DataFrame | str | Path, right: pd.DataFrame | str | Path) -> ProcCompareResult:
    """Compare two tables.

    When either input is a CSV path the comparison runs in CSV mode (chunked reading and hash partitions);
    otherwise both frames are compared in memory.

    Parameters
    ----------
    left : pandas.DataFrame, str or pathlib.Path
        The left table, or the path of a CSV file.
    right : pandas.DataFrame, str or pathlib.Path
        The right table, or the path of a CSV file.

    Returns
    -------
    ProcCompareResult
        The six result tables, plus the CSV paths and the Excel report path when they were written.

    Raises
    ------
    ValueError
        If neither ``key_cols`` nor ``row_order_compare`` is set, if a key column is missing from a table, or if a key
        is duplicated while ``duplicate_key_policy="raise"``.
    """
    key_cols = self._resolve_key_cols()
    if self._is_csv_like(left) or self._is_csv_like(right):
        result = self._run_csv(left, right, key_cols)
    else:
        left_df = self._prepare_dataframe(left, side=self.config.left_name, key_cols=key_cols)
        right_df = self._prepare_dataframe(right, side=self.config.right_name, key_cols=key_cols)
        schema = self._build_schema_summary(left_df, right_df, key_cols)
        result = self._compare_frames(left_df, right_df, key_cols, schema_summary=schema)

    output_paths: dict[str, str] = {}
    if self.config.write_outputs:
        output_paths = self._write_outputs(result)
        result.output_paths.update(output_paths)
    if self.config.write_excel:
        result.report_path = self._write_excel(result)
    return result

proc_compare

proc_compare(left: DataFrame | str | Path, right: DataFrame | str | Path, **kwargs: Any) -> ProcCompareResult

Convenience wrapper around :class:ProcCompareEngine.

Parameters:

Name	Type	Description	Default
left	(DataFrame, str or Path)	The left table, or the path of a CSV file.	required
right	(DataFrame, str or Path)	The right table, or the path of a CSV file.	required
**kwargs	Any	Fields of ProcCompareConfig, for example key_cols=["id"] or numeric_tol=1e-6.	{}

Returns:

Type	Description
ProcCompareResult	The result of ProcCompareEngine(ProcCompareConfig(**kwargs)).run(left, right).

Source code in Modeling_Tool/Core/Proc_Compare.py

def proc_compare(
    left: pd.DataFrame | str | Path,
    right: pd.DataFrame | str | Path,
    **kwargs: Any,
) -> ProcCompareResult:
    """Convenience wrapper around :class:`ProcCompareEngine`.

    Parameters
    ----------
    left : pandas.DataFrame, str or pathlib.Path
        The left table, or the path of a CSV file.
    right : pandas.DataFrame, str or pathlib.Path
        The right table, or the path of a CSV file.
    **kwargs
        Fields of ``ProcCompareConfig``, for example ``key_cols=["id"]`` or ``numeric_tol=1e-6``.

    Returns
    -------
    ProcCompareResult
        The result of ``ProcCompareEngine(ProcCompareConfig(**kwargs)).run(left, right)``.
    """
    return ProcCompareEngine(ProcCompareConfig(**kwargs)).run(left, right)

Slope Computation: Slope_Tool

Slope_Tool

SlopeCalculator

Slope calculator.

Provide several methods to compute the linear-regression slope of a data column, supporting: - sklearn LinearRegression - scipy.stats.linregress - numpy.polyfit - manual least squares

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the data	required
column	str	Name of the data column	required

Attributes:

Name	Type	Description
data	DataFrame	Input data
column	str	Column name
y	ndarray	Converted data array
x	ndarray	x-axis array (index)

Examples:

>>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
>>> calc = SlopeCalculator(df, 'values')
>>> calc.calculate_sklearn()
1.0
>>> calc.calculate_scipy()
(1.0, 1.0, 9.999999999999996e-08, 0.0)

Source code in Modeling_Tool/Core/Slope_Tool.py

class SlopeCalculator:
    """
    Slope calculator.

    Provide several methods to compute the linear-regression slope of a data column, supporting:
    - sklearn LinearRegression
    - scipy.stats.linregress
    - numpy.polyfit
    - manual least squares

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the data
    column : str
        Name of the data column

    Attributes
    ----------
    data : pandas.DataFrame
        Input data
    column : str
        Column name
    y : numpy.ndarray
        Converted data array
    x : numpy.ndarray
        x-axis array (index)

    Examples
    --------
    >>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
    >>> calc = SlopeCalculator(df, 'values')
    >>> calc.calculate_sklearn()
    1.0
    >>> calc.calculate_scipy()
    (1.0, 1.0, 9.999999999999996e-08, 0.0)
    """

    def __init__(self, data, column):
        """
        Initialize the slope calculator.

        Parameters
        ----------
        data : pandas.DataFrame
            DataFrame containing the data
        column : str
            Name of the data column
        """
        self.data = data
        self.column = column
        self.series = data[column]
        self.y = np.array(self.series)
        self.x = np.arange(len(self.series))

    def calculate_sklearn(self):
        """
        Compute the slope using sklearn LinearRegression.

        Returns
        -------
        float
            Slope of the linear regression
        """
        return calculate_slope_sklearn(self.data, self.column)

    def calculate_scipy(self):
        """
        Compute the slope using scipy.stats.linregress.

        Returns
        -------
        tuple
            Tuple (slope, r_value, p_value, std_err)
        """
        return calculate_slope_scipy(self.data, self.column)

    def calculate_numpy(self):
        """
        Compute the slope using numpy.polyfit.

        Returns
        -------
        float
            Slope of the linear regression
        """
        return calculate_slope_numpy(self.data, self.column)

    def calculate_manual(self):
        """
        Compute the slope using manual least squares.

        Returns
        -------
        float
            Slope of the linear regression
        """
        return calculate_slope_manual(self.data, self.column)

    def calculate_all(self):
        """
        Compute the slope using all methods.

        Returns
        -------
        dict
            Dictionary with the results of each method
        """
        results = {}

        # sklearn method
        results['sklearn'] = self.calculate_sklearn()

        # scipy method
        scipy_result = self.calculate_scipy()
        results['scipy_slope'] = scipy_result[0]
        results['scipy_r_value'] = scipy_result[1]
        results['scipy_p_value'] = scipy_result[2]
        results['scipy_std_err'] = scipy_result[3]

        # numpy method
        results['numpy'] = self.calculate_numpy()

        # manual method
        results['manual'] = self.calculate_manual()

        return results

    @staticmethod
    def calculate(data, column, method='sklearn'):
        """
        Compute the slope using the specified method (static method).

        Parameters
        ----------
        data : pandas.DataFrame
            DataFrame containing the data
        column : str
            Name of the data column
        method : str, default 'sklearn'
            Computation method; one of 'sklearn', 'scipy', 'numpy', 'manual'

        Returns
        -------
        float or tuple
            Slope value (scipy returns a tuple; the other methods return a float)

        Examples
        --------
        >>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
        >>> SlopeCalculator.calculate(df, 'values', method='numpy')
        1.0
        """
        calc = SlopeCalculator(data, column)

        if method == 'sklearn':
            return calc.calculate_sklearn()
        elif method == 'scipy':
            return calc.calculate_scipy()
        elif method == 'numpy':
            return calc.calculate_numpy()
        elif method == 'manual':
            return calc.calculate_manual()
        else:
            raise ValueError(f"Unsupported method: {method}. Choose one of: 'sklearn', 'scipy', 'numpy', 'manual'")

calculate_sklearn

calculate_sklearn()

Compute the slope using sklearn LinearRegression.

Returns:

Type	Description
float	Slope of the linear regression

Source code in Modeling_Tool/Core/Slope_Tool.py

def calculate_sklearn(self):
    """
    Compute the slope using sklearn LinearRegression.

    Returns
    -------
    float
        Slope of the linear regression
    """
    return calculate_slope_sklearn(self.data, self.column)

calculate_scipy

calculate_scipy()

Compute the slope using scipy.stats.linregress.

Returns:

Type	Description
tuple	Tuple (slope, r_value, p_value, std_err)

Source code in Modeling_Tool/Core/Slope_Tool.py

def calculate_scipy(self):
    """
    Compute the slope using scipy.stats.linregress.

    Returns
    -------
    tuple
        Tuple (slope, r_value, p_value, std_err)
    """
    return calculate_slope_scipy(self.data, self.column)

calculate_numpy

calculate_numpy()

Compute the slope using numpy.polyfit.

Returns:

Type	Description
float	Slope of the linear regression

Source code in Modeling_Tool/Core/Slope_Tool.py

def calculate_numpy(self):
    """
    Compute the slope using numpy.polyfit.

    Returns
    -------
    float
        Slope of the linear regression
    """
    return calculate_slope_numpy(self.data, self.column)

calculate_manual

calculate_manual()

Compute the slope using manual least squares.

Returns:

Type	Description
float	Slope of the linear regression

Source code in Modeling_Tool/Core/Slope_Tool.py

def calculate_manual(self):
    """
    Compute the slope using manual least squares.

    Returns
    -------
    float
        Slope of the linear regression
    """
    return calculate_slope_manual(self.data, self.column)

calculate_all

calculate_all()

Compute the slope using all methods.

Returns:

Type	Description
dict	Dictionary with the results of each method

Source code in Modeling_Tool/Core/Slope_Tool.py

def calculate_all(self):
    """
    Compute the slope using all methods.

    Returns
    -------
    dict
        Dictionary with the results of each method
    """
    results = {}

    # sklearn method
    results['sklearn'] = self.calculate_sklearn()

    # scipy method
    scipy_result = self.calculate_scipy()
    results['scipy_slope'] = scipy_result[0]
    results['scipy_r_value'] = scipy_result[1]
    results['scipy_p_value'] = scipy_result[2]
    results['scipy_std_err'] = scipy_result[3]

    # numpy method
    results['numpy'] = self.calculate_numpy()

    # manual method
    results['manual'] = self.calculate_manual()

    return results

calculate staticmethod

calculate(data, column, method='sklearn')

Compute the slope using the specified method (static method).

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the data	required
column	str	Name of the data column	required
method	str	Computation method; one of 'sklearn', 'scipy', 'numpy', 'manual'	'sklearn'

Returns:

Type	Description
float or tuple	Slope value (scipy returns a tuple; the other methods return a float)

Examples:

>>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
>>> SlopeCalculator.calculate(df, 'values', method='numpy')
1.0

Source code in Modeling_Tool/Core/Slope_Tool.py

@staticmethod
def calculate(data, column, method='sklearn'):
    """
    Compute the slope using the specified method (static method).

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the data
    column : str
        Name of the data column
    method : str, default 'sklearn'
        Computation method; one of 'sklearn', 'scipy', 'numpy', 'manual'

    Returns
    -------
    float or tuple
        Slope value (scipy returns a tuple; the other methods return a float)

    Examples
    --------
    >>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
    >>> SlopeCalculator.calculate(df, 'values', method='numpy')
    1.0
    """
    calc = SlopeCalculator(data, column)

    if method == 'sklearn':
        return calc.calculate_sklearn()
    elif method == 'scipy':
        return calc.calculate_scipy()
    elif method == 'numpy':
        return calc.calculate_numpy()
    elif method == 'manual':
        return calc.calculate_manual()
    else:
        raise ValueError(f"Unsupported method: {method}. Choose one of: 'sklearn', 'scipy', 'numpy', 'manual'")

calculate_slope_sklearn

calculate_slope_sklearn(data, column)

Compute the slope of a data column using scikit-learn's LinearRegression.

Based on ordinary least squares: fit the data points with a LinearRegression model and return the slope coefficient of the linear regression.

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the data	required
column	str	Name of the data column	required

Returns:

Type	Description
float	Slope of the linear regression

Examples:

>>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
>>> calculate_slope_sklearn(df, 'values')
1.0

Source code in Modeling_Tool/Core/Slope_Tool.py

def calculate_slope_sklearn(data, column):
    """
    Compute the slope of a data column using scikit-learn's LinearRegression.

    Based on ordinary least squares: fit the data points with a LinearRegression model
    and return the slope coefficient of the linear regression.

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the data
    column : str
        Name of the data column

    Returns
    -------
    float
        Slope of the linear regression

    Examples
    --------
    >>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
    >>> calculate_slope_sklearn(df, 'values')
    1.0
    """

    from sklearn.linear_model import LinearRegression

    series = data[column]
    # Make sure the data is a NumPy array
    y = np.array(series).reshape(-1, 1)

    # Create the x axis (index)
    x = np.arange(len(series)).reshape(-1, 1)

    # Create and fit the linear regression model
    model = LinearRegression()
    model.fit(x, y)

    # Get the slope
    slope = model.coef_[0][0]

    return slope

calculate_slope_scipy

calculate_slope_scipy(data, column)

Compute the slope of a data column using SciPy's linregress function.

Based on ordinary least squares: fit the data points with scipy.stats.linregress and return the slope along with additional statistics.

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the data	required
column	str	Name of the data column	required

Returns:

Type	Description
tuple	Tuple (slope, r_value, p_value, std_err) containing: - slope: slope value - r_value: correlation coefficient - p_value: p-value - std_err: standard error

Examples:

>>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
>>> slope, r, p, se = calculate_slope_scipy(df, 'values')
>>> print(f"Slope: {slope}, correlation: {r}")
Slope: 1.0, correlation: 1.0

Source code in Modeling_Tool/Core/Slope_Tool.py

def calculate_slope_scipy(data, column):
    """
    Compute the slope of a data column using SciPy's linregress function.

    Based on ordinary least squares: fit the data points with scipy.stats.linregress
    and return the slope along with additional statistics.

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the data
    column : str
        Name of the data column

    Returns
    -------
    tuple
        Tuple (slope, r_value, p_value, std_err) containing:
        - slope: slope value
        - r_value: correlation coefficient
        - p_value: p-value
        - std_err: standard error

    Examples
    --------
    >>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
    >>> slope, r, p, se = calculate_slope_scipy(df, 'values')
    >>> print(f"Slope: {slope}, correlation: {r}")
    Slope: 1.0, correlation: 1.0
    """

    from scipy import stats

    series = data[column]
    # Make sure the data is a NumPy array
    y = np.array(series)

    # Create the x axis (index)
    x = np.arange(len(y))

    # Run the linear regression
    slope, intercept, r_value, p_value, std_err = stats.linregress(x, y)

    return slope, r_value, p_value, std_err

calculate_slope_numpy

calculate_slope_numpy(data, column)

Compute the slope of a data column using NumPy's polyfit function.

Fit a first-degree polynomial with numpy.polyfit and return the slope of the linear regression.

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the data	required
column	str	Name of the data column	required

Returns:

Type	Description
float	Slope of the linear regression

Examples:

>>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
>>> calculate_slope_numpy(df, 'values')
1.0

Source code in Modeling_Tool/Core/Slope_Tool.py

def calculate_slope_numpy(data, column):
    """
    Compute the slope of a data column using NumPy's polyfit function.

    Fit a first-degree polynomial with numpy.polyfit
    and return the slope of the linear regression.

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the data
    column : str
        Name of the data column

    Returns
    -------
    float
        Slope of the linear regression

    Examples
    --------
    >>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
    >>> calculate_slope_numpy(df, 'values')
    1.0
    """

    import numpy as np

    series = data[column]
    # Make sure the data is a NumPy array
    y = np.array(series)

    # Create the x axis (index)
    x = np.arange(len(y))

    # Fit a first-degree polynomial (linear regression); returns the slope and intercept
    slope, intercept = np.polyfit(x, y, 1)

    return slope

calculate_slope_manual

calculate_slope_manual(data, column)

Compute the slope of a data column manually with ordinary least squares.

Implement the least-squares formula by hand to compute the slope of the linear regression: slope = Σ((x - x_mean) * (y - y_mean)) / Σ((x - x_mean)²)

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the data	required
column	str	Name of the data column	required

Returns:

Type	Description
float	Slope of the linear regression

Examples:

>>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
>>> calculate_slope_manual(df, 'values')
1.0

Source code in Modeling_Tool/Core/Slope_Tool.py

def calculate_slope_manual(data, column):
    """
    Compute the slope of a data column manually with ordinary least squares.

    Implement the least-squares formula by hand to compute the slope of the linear regression:
    slope = Σ((x - x_mean) * (y - y_mean)) / Σ((x - x_mean)²)

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the data
    column : str
        Name of the data column

    Returns
    -------
    float
        Slope of the linear regression

    Examples
    --------
    >>> df = pd.DataFrame({'values': [1, 2, 3, 4, 5]})
    >>> calculate_slope_manual(df, 'values')
    1.0
    """

    series = data[column]

    # Make sure the data is a NumPy array
    y = np.array(series)

    # Create the x axis (index)
    x = np.arange(len(y))

    # Compute the means of x and y
    x_mean = np.mean(x)
    y_mean = np.mean(y)

    # Compute the slope and intercept
    numerator = np.sum((x - x_mean) * (y - y_mean))
    denominator = np.sum((x - x_mean) ** 2)

    slope = numerator / denominator

    return slope

General Utilities: utils

utils

DataFrameProcessor

Utility class for processing DataFrames.

Provide a unified interface for DataFrame operations, including column operations, row filtering and type conversion.

Parameters:

Name	Type	Description	Default
data	DataFrame	The DataFrame to process.	required

Examples:

>>> processor = DataFrameProcessor(df)
>>> processor.move_column('col_a', 0)
>>> processor.convert_colnames('lower')

Source code in Modeling_Tool/Core/utils.py

class DataFrameProcessor:
    """
    Utility class for processing DataFrames.

    Provide a unified interface for DataFrame operations, including column operations, row filtering and
    type conversion.

    Parameters
    ----------
    data : pandas.DataFrame
        The DataFrame to process.

    Examples
    --------
    >>> processor = DataFrameProcessor(df)
    >>> processor.move_column('col_a', 0)
    >>> processor.convert_colnames('lower')
    """

    def __init__(self, data):
        """
        Initialize the DataFrame processor.

        Parameters
        ----------
        data : pandas.DataFrame
            The DataFrame to process.
        """
        self.data = data

    def move_column(self, colname, idx, return_kDF=True, h2o_frame=False):
        """
        Move a column to a given position.

        Parameters
        ----------
        colname : str
            Name of the column to move.
        idx : int
            Target position (0-based) of the column, with ``list.insert`` semantics.
        return_kDF : bool, default True
            Whether to return a kDataFrame. Ignored (treated as False) when ``h2o_frame=True``.
        h2o_frame : bool, default False
            Whether the input is an H2OFrame.

        Returns
        -------
        DataFrame or kDataFrame
            The data of the processor with the columns reordered (see the module-level ``move_column``, which this method
            calls; ``h2o`` must be installed).
        """
        return move_column(self.data, colname, idx, return_kDF, h2o_frame)

    def convert_colnames(self, how="lowercase", return_kDF=True):
        """
        Convert the case of the column names.

        Parameters
        ----------
        how : str, default "lowercase"
            Conversion method: one of 'lower'/'lowercase', 'upper'/'uppercase', 'cap'/'capitalize'.
        return_kDF : bool, default True
            Whether to return a kDataFrame.

        Returns
        -------
        DataFrame or kDataFrame
            The data with the converted column names. The columns of the processor's own ``data`` are renamed in place (see the
            module-level ``convert_colnames``, which this method calls).
        """
        return convert_colnames(self.data, how, return_kDF)

    def col_filter_regex(self, regex, case_sensitive=True, h2o_frame=False, return_kDF=True):
        """
        Filter the columns by a regular expression.

        Parameters
        ----------
        regex : str
            Regular expression.
        case_sensitive : bool, default True
            Whether matching is case-sensitive (ignored when ``h2o_frame=True``).
        h2o_frame : bool, default False
            Whether the input is an H2OFrame.
        return_kDF : bool, default True
            Whether to return a kDataFrame. Ignored (treated as False) when ``h2o_frame=True``.

        Returns
        -------
        DataFrame or kDataFrame
        """
        return col_filter_regex(self.data, regex, case_sensitive, h2o_frame, return_kDF)

    def row_filter_regex(self, col, regex, case_sensitive=True, as_index=False, return_kDF=True):
        """
        Filter the rows by a regular expression.

        Parameters
        ----------
        col : str
            Name of the column to filter on.
        regex : str
            Regular expression.
        case_sensitive : bool, default True
            Whether matching is case-sensitive.
        as_index : bool, default False
            Whether to use the filter column as the index. Only honored when ``return_kDF=False``.
        return_kDF : bool, default True
            Whether to return a kDataFrame. When True, ``as_index`` has no effect.

        Returns
        -------
        DataFrame or kDataFrame
        """
        return row_filter_regex(self.data, col, regex, case_sensitive, as_index, return_kDF)

    def get_dtypes(self, outputFile=None, ck_format=False):
        """
        Get the data types.

        Parameters
        ----------
        outputFile : str, optional
            Output file path. If given, the table is also written there as CSV, without the index and without a header row.
        ck_format : bool, default False
            Reserved for a ClickHouse dtype mapping that SMF does not ship: ``ck_format=True`` raises
            ``NotImplementedError``, so leave it at False.

        Returns
        -------
        pandas.DataFrame
            The column names and data types (see ``get_dtypes_file``, which this method calls).
        """
        return get_dtypes_file(self.data, outputFile, ck_format)

    def drop_tmp_cols(self, drop_list=['py_inserttime']):
        """
        Drop temporary columns.

        Parameters
        ----------
        drop_list : list, default ['py_inserttime']
            List of columns to drop. Names that are not columns of the data are ignored.

        Returns
        -------
        DataFrame
        """
        return drop_tmp_cols(self.data, drop_list)

    def to_bool_str(self):
        """
        Convert boolean columns to strings.

        Returns
        -------
        DataFrame
        """
        return bool_to_str(self.data)

move_column

move_column(colname, idx, return_kDF=True, h2o_frame=False)

Move a column to a given position.

Parameters:

Name	Type	Description	Default
colname	str	Name of the column to move.	required
idx	int	Target position (0-based) of the column, with list.insert semantics.	required
return_kDF	bool	Whether to return a kDataFrame. Ignored (treated as False) when h2o_frame=True.	True
h2o_frame	bool	Whether the input is an H2OFrame.	False

Returns:

Type	Description
DataFrame or kDataFrame	The data of the processor with the columns reordered (see the module-level move_column, which this method calls; h2o must be installed).

Source code in Modeling_Tool/Core/utils.py

def move_column(self, colname, idx, return_kDF=True, h2o_frame=False):
    """
    Move a column to a given position.

    Parameters
    ----------
    colname : str
        Name of the column to move.
    idx : int
        Target position (0-based) of the column, with ``list.insert`` semantics.
    return_kDF : bool, default True
        Whether to return a kDataFrame. Ignored (treated as False) when ``h2o_frame=True``.
    h2o_frame : bool, default False
        Whether the input is an H2OFrame.

    Returns
    -------
    DataFrame or kDataFrame
        The data of the processor with the columns reordered (see the module-level ``move_column``, which this method
        calls; ``h2o`` must be installed).
    """
    return move_column(self.data, colname, idx, return_kDF, h2o_frame)

convert_colnames

convert_colnames(how='lowercase', return_kDF=True)

Convert the case of the column names.

Parameters:

Name	Type	Description	Default
how	str	Conversion method: one of 'lower'/'lowercase', 'upper'/'uppercase', 'cap'/'capitalize'.	"lowercase"
return_kDF	bool	Whether to return a kDataFrame.	True

Returns:

Type	Description
DataFrame or kDataFrame	The data with the converted column names. The columns of the processor's own data are renamed in place (see the module-level convert_colnames, which this method calls).

Source code in Modeling_Tool/Core/utils.py

def convert_colnames(self, how="lowercase", return_kDF=True):
    """
    Convert the case of the column names.

    Parameters
    ----------
    how : str, default "lowercase"
        Conversion method: one of 'lower'/'lowercase', 'upper'/'uppercase', 'cap'/'capitalize'.
    return_kDF : bool, default True
        Whether to return a kDataFrame.

    Returns
    -------
    DataFrame or kDataFrame
        The data with the converted column names. The columns of the processor's own ``data`` are renamed in place (see the
        module-level ``convert_colnames``, which this method calls).
    """
    return convert_colnames(self.data, how, return_kDF)

col_filter_regex

col_filter_regex(regex, case_sensitive=True, h2o_frame=False, return_kDF=True)

Filter the columns by a regular expression.

Parameters:

Name	Type	Description	Default
regex	str	Regular expression.	required
case_sensitive	bool	Whether matching is case-sensitive (ignored when h2o_frame=True).	True
h2o_frame	bool	Whether the input is an H2OFrame.	False
return_kDF	bool	Whether to return a kDataFrame. Ignored (treated as False) when h2o_frame=True.	True

Returns:

Type	Description
DataFrame or kDataFrame

Source code in Modeling_Tool/Core/utils.py

def col_filter_regex(self, regex, case_sensitive=True, h2o_frame=False, return_kDF=True):
    """
    Filter the columns by a regular expression.

    Parameters
    ----------
    regex : str
        Regular expression.
    case_sensitive : bool, default True
        Whether matching is case-sensitive (ignored when ``h2o_frame=True``).
    h2o_frame : bool, default False
        Whether the input is an H2OFrame.
    return_kDF : bool, default True
        Whether to return a kDataFrame. Ignored (treated as False) when ``h2o_frame=True``.

    Returns
    -------
    DataFrame or kDataFrame
    """
    return col_filter_regex(self.data, regex, case_sensitive, h2o_frame, return_kDF)

row_filter_regex

row_filter_regex(col, regex, case_sensitive=True, as_index=False, return_kDF=True)

Filter the rows by a regular expression.

Parameters:

Name	Type	Description	Default
col	str	Name of the column to filter on.	required
regex	str	Regular expression.	required
case_sensitive	bool	Whether matching is case-sensitive.	True
as_index	bool	Whether to use the filter column as the index. Only honored when return_kDF=False.	False
return_kDF	bool	Whether to return a kDataFrame. When True, as_index has no effect.	True

Returns:

Type	Description
DataFrame or kDataFrame

Source code in Modeling_Tool/Core/utils.py

def row_filter_regex(self, col, regex, case_sensitive=True, as_index=False, return_kDF=True):
    """
    Filter the rows by a regular expression.

    Parameters
    ----------
    col : str
        Name of the column to filter on.
    regex : str
        Regular expression.
    case_sensitive : bool, default True
        Whether matching is case-sensitive.
    as_index : bool, default False
        Whether to use the filter column as the index. Only honored when ``return_kDF=False``.
    return_kDF : bool, default True
        Whether to return a kDataFrame. When True, ``as_index`` has no effect.

    Returns
    -------
    DataFrame or kDataFrame
    """
    return row_filter_regex(self.data, col, regex, case_sensitive, as_index, return_kDF)

get_dtypes

get_dtypes(outputFile=None, ck_format=False)

Get the data types.

Parameters:

Name	Type	Description	Default
outputFile	str	Output file path. If given, the table is also written there as CSV, without the index and without a header row.	None
ck_format	bool	Reserved for a ClickHouse dtype mapping that SMF does not ship: ck_format=True raises NotImplementedError, so leave it at False.	False

Returns:

Type	Description
DataFrame	The column names and data types (see get_dtypes_file, which this method calls).

Source code in Modeling_Tool/Core/utils.py

def get_dtypes(self, outputFile=None, ck_format=False):
    """
    Get the data types.

    Parameters
    ----------
    outputFile : str, optional
        Output file path. If given, the table is also written there as CSV, without the index and without a header row.
    ck_format : bool, default False
        Reserved for a ClickHouse dtype mapping that SMF does not ship: ``ck_format=True`` raises
        ``NotImplementedError``, so leave it at False.

    Returns
    -------
    pandas.DataFrame
        The column names and data types (see ``get_dtypes_file``, which this method calls).
    """
    return get_dtypes_file(self.data, outputFile, ck_format)

drop_tmp_cols

drop_tmp_cols(drop_list=['py_inserttime'])

Drop temporary columns.

Parameters:

Name	Type	Description	Default
drop_list	list	List of columns to drop. Names that are not columns of the data are ignored.	['py_inserttime']

Returns:

Type	Description
DataFrame

Source code in Modeling_Tool/Core/utils.py

def drop_tmp_cols(self, drop_list=['py_inserttime']):
    """
    Drop temporary columns.

    Parameters
    ----------
    drop_list : list, default ['py_inserttime']
        List of columns to drop. Names that are not columns of the data are ignored.

    Returns
    -------
    DataFrame
    """
    return drop_tmp_cols(self.data, drop_list)

to_bool_str

to_bool_str()

Convert boolean columns to strings.

Returns:

Type	Description
DataFrame

Source code in Modeling_Tool/Core/utils.py

def to_bool_str(self):
    """
    Convert boolean columns to strings.

    Returns
    -------
    DataFrame
    """
    return bool_to_str(self.data)

FilePathManager

Utility class for managing file paths.

Provide path operations, file listing and directory creation.

Attributes:

Name	Type	Description
base_path	str	Base path given to the constructor (the current working directory when none is given). It is only stored: none of the methods uses it.

Examples:

>>> fpm = FilePathManager('/base/path')
>>> fpm.get_filenames('/base/path', '.*\.csv')
>>> fpm.mkdir('output')

Source code in Modeling_Tool/Core/utils.py

class FilePathManager:
    """
    Utility class for managing file paths.

    Provide path operations, file listing and directory creation.

    Attributes
    ----------
    base_path : str
        Base path given to the constructor (the current working directory when none is given). It is only stored: none of
        the methods uses it.

    Examples
    --------
    >>> fpm = FilePathManager('/base/path')
    >>> fpm.get_filenames('/base/path', '.*\\.csv')
    >>> fpm.mkdir('output')
    """

    def __init__(self, base_path=None):
        """
        Initialize the path manager.

        Parameters
        ----------
        base_path : str, optional
            Base path. The current working directory is used when it is not given (None or empty).
        """
        self.base_path = base_path or os.getcwd()

    def get_filenames(self, path, regex):
        """
        Get the list of matching file names.

        Parameters
        ----------
        path : str
            Directory path.
        regex : str
            Regular expression.

        Returns
        -------
        list
            The matching file names, searched recursively (see the module-level ``get_filenames``).
        """
        return get_filenames(path, regex)

    def add_suffix(self, file, suffix="_cut"):
        """
        Add a suffix to a file path.

        Parameters
        ----------
        file : str
            File path.
        suffix : str, default "_cut"
            Suffix.

        Returns
        -------
        str
        """
        return add_path_suffix(file, suffix)

    def mkdir(self, folder_path, replace=False):
        """
        Create a directory.

        Parameters
        ----------
        folder_path : str
            Directory path.
        replace : bool, default False
            Whether to replace an existing directory. An existing directory is not deleted or emptied (see the module-level
            ``mkdir_if_not_exist``, which this method calls).

        Returns
        -------
        int
            Status code: 0 means success, 1 means the directory already exists (``None`` when ``folder_path`` is ``None``).
        """
        return mkdir_if_not_exist(folder_path, replace)

    def get_curr_abs_path(self, path):
        """
        Get the absolute path.

        Parameters
        ----------
        path : str
            Relative path, resolved against the directory of the module ``Modeling_Tool/Core/utils.py`` (not against
            ``base_path``).

        Returns
        -------
        str
        """
        return get_curr_abs_path(path)

get_filenames

get_filenames(path, regex)

Get the list of matching file names.

Parameters:

Name	Type	Description	Default
path	str	Directory path.	required
regex	str	Regular expression.	required

Returns:

Type	Description
list	The matching file names, searched recursively (see the module-level get_filenames).

Source code in Modeling_Tool/Core/utils.py

def get_filenames(self, path, regex):
    """
    Get the list of matching file names.

    Parameters
    ----------
    path : str
        Directory path.
    regex : str
        Regular expression.

    Returns
    -------
    list
        The matching file names, searched recursively (see the module-level ``get_filenames``).
    """
    return get_filenames(path, regex)

add_suffix

add_suffix(file, suffix='_cut')

Add a suffix to a file path.

Parameters:

Name	Type	Description	Default
file	str	File path.	required
suffix	str	Suffix.	"_cut"

Returns:

Type	Description
str

Source code in Modeling_Tool/Core/utils.py

def add_suffix(self, file, suffix="_cut"):
    """
    Add a suffix to a file path.

    Parameters
    ----------
    file : str
        File path.
    suffix : str, default "_cut"
        Suffix.

    Returns
    -------
    str
    """
    return add_path_suffix(file, suffix)

mkdir

mkdir(folder_path, replace=False)

Create a directory.

Parameters:

Name	Type	Description	Default
folder_path	str	Directory path.	required
replace	bool	Whether to replace an existing directory. An existing directory is not deleted or emptied (see the module-level mkdir_if_not_exist, which this method calls).	False

Returns:

Type	Description
int	Status code: 0 means success, 1 means the directory already exists (None when folder_path is None).

Source code in Modeling_Tool/Core/utils.py

def mkdir(self, folder_path, replace=False):
    """
    Create a directory.

    Parameters
    ----------
    folder_path : str
        Directory path.
    replace : bool, default False
        Whether to replace an existing directory. An existing directory is not deleted or emptied (see the module-level
        ``mkdir_if_not_exist``, which this method calls).

    Returns
    -------
    int
        Status code: 0 means success, 1 means the directory already exists (``None`` when ``folder_path`` is ``None``).
    """
    return mkdir_if_not_exist(folder_path, replace)

get_curr_abs_path

get_curr_abs_path(path)

Get the absolute path.

Parameters:

Name	Type	Description	Default
path	str	Relative path, resolved against the directory of the module Modeling_Tool/Core/utils.py (not against base_path).	required

Returns:

Type	Description
str

Source code in Modeling_Tool/Core/utils.py

def get_curr_abs_path(self, path):
    """
    Get the absolute path.

    Parameters
    ----------
    path : str
        Relative path, resolved against the directory of the module ``Modeling_Tool/Core/utils.py`` (not against
        ``base_path``).

    Returns
    -------
    str
    """
    return get_curr_abs_path(path)

DateTimeUtils

Utility class for dates and times.

Provide convenient date- and time-related methods.

Examples:

>>> dt_utils = DateTimeUtils()
>>> dt_utils.get_curr_datetime('-')
'20250330-143624'
>>> dt_utils.get_last_vintage()
'202502'

Source code in Modeling_Tool/Core/utils.py

class DateTimeUtils:
    """
    Utility class for dates and times.

    Provide convenient date- and time-related methods.

    Examples
    --------
    >>> dt_utils = DateTimeUtils()
    >>> dt_utils.get_curr_datetime('-')
    '20250330-143624'
    >>> dt_utils.get_last_vintage()
    '202502'
    """

    def get_curr_datetime(self, sep=''):
        """
        Get the current date and time.

        Parameters
        ----------
        sep : str, default ''
            Separator.

        Returns
        -------
        str
        """
        return get_curr_datetime(sep)

    def get_buffer_date(self, start_date):
        """
        Get the buffer date.

        Parameters
        ----------
        start_date : str
            Start date.

        Returns
        -------
        str
        """
        return get_buffer_date(start_date)

    def get_quarter(self, strDate):
        """
        Get the quarter.

        Parameters
        ----------
        strDate : str
            Date string.

        Returns
        -------
        int
        """
        return get_quarter(strDate)

    def get_last_vintage(self):
        """
        Get the vintage of the previous month.

        Returns
        -------
        str
        """
        return get_last_vintage()

    def last_month_vintage(self, year, month, day):
        """
        Get the vintage of the previous month.

        Parameters
        ----------
        year : int
        month : int
        day : int

        Returns
        -------
        int
        """
        return last_Month_Vintage(year, month, day)

    def get_valid_vintages(self, sVintage, eVintage):
        """
        Get the list of valid vintages.

        Parameters
        ----------
        sVintage : int
        eVintage : int

        Returns
        -------
        list
        """
        return get_valid_vintages(sVintage, eVintage)

get_curr_datetime

get_curr_datetime(sep='')

Get the current date and time.

Parameters:

Name	Type	Description	Default
sep	str	Separator.	''

Returns:

Type	Description
str

Source code in Modeling_Tool/Core/utils.py

def get_curr_datetime(self, sep=''):
    """
    Get the current date and time.

    Parameters
    ----------
    sep : str, default ''
        Separator.

    Returns
    -------
    str
    """
    return get_curr_datetime(sep)

get_buffer_date

get_buffer_date(start_date)

Get the buffer date.

Parameters:

Name	Type	Description	Default
start_date	str	Start date.	required

Returns:

Type	Description
str

Source code in Modeling_Tool/Core/utils.py

def get_buffer_date(self, start_date):
    """
    Get the buffer date.

    Parameters
    ----------
    start_date : str
        Start date.

    Returns
    -------
    str
    """
    return get_buffer_date(start_date)

get_quarter

get_quarter(strDate)

Get the quarter.

Parameters:

Name	Type	Description	Default
strDate	str	Date string.	required

Returns:

Type	Description
int

Source code in Modeling_Tool/Core/utils.py

def get_quarter(self, strDate):
    """
    Get the quarter.

    Parameters
    ----------
    strDate : str
        Date string.

    Returns
    -------
    int
    """
    return get_quarter(strDate)

get_last_vintage

get_last_vintage()

Get the vintage of the previous month.

Returns:

Type	Description
str

Source code in Modeling_Tool/Core/utils.py

def get_last_vintage(self):
    """
    Get the vintage of the previous month.

    Returns
    -------
    str
    """
    return get_last_vintage()

last_month_vintage

last_month_vintage(year, month, day)

Get the vintage of the previous month.

Parameters:

Name	Type	Description	Default
year	int		required
month	int		required
day	int		required

Returns:

Type	Description
int

Source code in Modeling_Tool/Core/utils.py

def last_month_vintage(self, year, month, day):
    """
    Get the vintage of the previous month.

    Parameters
    ----------
    year : int
    month : int
    day : int

    Returns
    -------
    int
    """
    return last_Month_Vintage(year, month, day)

get_valid_vintages

get_valid_vintages(sVintage, eVintage)

Get the list of valid vintages.

Parameters:

Name	Type	Description	Default
sVintage	int		required
eVintage	int		required

Returns:

Type	Description
list

Source code in Modeling_Tool/Core/utils.py

def get_valid_vintages(self, sVintage, eVintage):
    """
    Get the list of valid vintages.

    Parameters
    ----------
    sVintage : int
    eVintage : int

    Returns
    -------
    list
    """
    return get_valid_vintages(sVintage, eVintage)

WOEIVCalculator

Utility class for computing WOE and IV.

Provide the WOE and IV calculations commonly used in credit scoring.

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the proportions.	required
bad_pct_col	str	Name of the column holding the bad-sample proportion.	required
good_pct_col	str	Name of the column holding the good-sample proportion.	required

Examples:

>>> calc = WOEIVCalculator(df, 'bad_pct', 'good_pct')
>>> calc.calc_woe()
>>> calc.calc_iv()
>>> woe, iv = calc.calc_both()

Source code in Modeling_Tool/Core/utils.py

class WOEIVCalculator:
    """
    Utility class for computing WOE and IV.

    Provide the WOE and IV calculations commonly used in credit scoring.

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the proportions.
    bad_pct_col : str
        Name of the column holding the bad-sample proportion.
    good_pct_col : str
        Name of the column holding the good-sample proportion.

    Examples
    --------
    >>> calc = WOEIVCalculator(df, 'bad_pct', 'good_pct')
    >>> calc.calc_woe()
    >>> calc.calc_iv()
    >>> woe, iv = calc.calc_both()
    """

    def __init__(self, data, bad_pct_col, good_pct_col):
        """
        Initialize the WOE/IV calculator.

        Parameters
        ----------
        data : pandas.DataFrame
            DataFrame containing the proportions.
        bad_pct_col : str
            Name of the column holding the bad-sample proportion.
        good_pct_col : str
            Name of the column holding the good-sample proportion.
        """
        self.data = data
        self.bad_pct_col = bad_pct_col
        self.good_pct_col = good_pct_col

    def calc_woe(self, fillna=True):
        """
        Compute the WOE values.

        Parameters
        ----------
        fillna : bool, default True
            Passed to ``calc_woe`` as ``fillwoe``: the value returned when the data has no rows (0 if True, NaN if False). It
            does not fill rows with a zero proportion, which give ``inf``, ``-inf`` or NaN.

        Returns
        -------
        float or Series
        """
        return calc_woe(self.data, self.bad_pct_col, self.good_pct_col, fillna)

    def calc_iv(self, fillna=True):
        """
        Compute the IV values.

        Parameters
        ----------
        fillna : bool, default True
            Passed to ``calc_iv`` as ``filliv``: the value returned when the data has no rows (0 if True, NaN if False). It
            does not fill rows with a zero proportion, which give ``inf`` or NaN.

        Returns
        -------
        float or Series
        """
        return calc_iv(self.data, self.bad_pct_col, self.good_pct_col, fillna)

    def calc_both(self, fillna=True):
        """
        Compute the WOE and IV together.

        Parameters
        ----------
        fillna : bool, default True
            Used as both ``fillwoe`` and ``filliv``: the value returned when the data has no rows (0 if True, NaN if False). It
            does not fill rows with a zero proportion.

        Returns
        -------
        tuple
            ``(woe, iv)``: the WOE and the IV, each a Series aligned with the data (the scalars 0 / NaN when the data is empty).
        """
        return _calc_woe_iv_values(
            self.data,
            self.bad_pct_col,
            self.good_pct_col,
            fillwoe=fillna,
            filliv=fillna,
        )

calc_woe

calc_woe(fillna=True)

Compute the WOE values.

Parameters:

Name	Type	Description	Default
fillna	bool	Passed to calc_woe as fillwoe: the value returned when the data has no rows (0 if True, NaN if False). It does not fill rows with a zero proportion, which give inf, -inf or NaN.	True

Returns:

Type	Description
float or Series

Source code in Modeling_Tool/Core/utils.py

def calc_woe(self, fillna=True):
    """
    Compute the WOE values.

    Parameters
    ----------
    fillna : bool, default True
        Passed to ``calc_woe`` as ``fillwoe``: the value returned when the data has no rows (0 if True, NaN if False). It
        does not fill rows with a zero proportion, which give ``inf``, ``-inf`` or NaN.

    Returns
    -------
    float or Series
    """
    return calc_woe(self.data, self.bad_pct_col, self.good_pct_col, fillna)

calc_iv

calc_iv(fillna=True)

Compute the IV values.

Parameters:

Name	Type	Description	Default
fillna	bool	Passed to calc_iv as filliv: the value returned when the data has no rows (0 if True, NaN if False). It does not fill rows with a zero proportion, which give inf or NaN.	True

Returns:

Type	Description
float or Series

Source code in Modeling_Tool/Core/utils.py

def calc_iv(self, fillna=True):
    """
    Compute the IV values.

    Parameters
    ----------
    fillna : bool, default True
        Passed to ``calc_iv`` as ``filliv``: the value returned when the data has no rows (0 if True, NaN if False). It
        does not fill rows with a zero proportion, which give ``inf`` or NaN.

    Returns
    -------
    float or Series
    """
    return calc_iv(self.data, self.bad_pct_col, self.good_pct_col, fillna)

calc_both

calc_both(fillna=True)

Compute the WOE and IV together.

Parameters:

Name	Type	Description	Default
fillna	bool	Used as both fillwoe and filliv: the value returned when the data has no rows (0 if True, NaN if False). It does not fill rows with a zero proportion.	True

Returns:

Type	Description
tuple	(woe, iv): the WOE and the IV, each a Series aligned with the data (the scalars 0 / NaN when the data is empty).

Source code in Modeling_Tool/Core/utils.py

def calc_both(self, fillna=True):
    """
    Compute the WOE and IV together.

    Parameters
    ----------
    fillna : bool, default True
        Used as both ``fillwoe`` and ``filliv``: the value returned when the data has no rows (0 if True, NaN if False). It
        does not fill rows with a zero proportion.

    Returns
    -------
    tuple
        ``(woe, iv)``: the WOE and the IV, each a Series aligned with the data (the scalars 0 / NaN when the data is empty).
    """
    return _calc_woe_iv_values(
        self.data,
        self.bad_pct_col,
        self.good_pct_col,
        fillwoe=fillna,
        filliv=fillna,
    )

bucket_by_cond

bucket_by_cond(df: DataFrame, cond_dict: dict, colname: str, drop_unmatched: bool = True, default=nan) -> DataFrame

Bucket the rows of a DataFrame by a dictionary of query conditions and tag each bucket with its label.

Parameters:

Name	Type	Description	Default
df	DataFrame	Original data.	required
cond_dict	dict	Mapping of {label: query condition string}.	required
colname	str	Name of the new label column.	required
drop_unmatched	bool	If True, drop rows that match no condition (original behavior); if False, keep all rows.	True
default	scalar	Value assigned to unmatched rows (only used when drop_unmatched=False).	np.nan

Returns:

Type	Description
DataFrame	Data with the new label column.

Notes

The conditions are evaluated with DataFrame.query when drop_unmatched=True and with DataFrame.eval when drop_unmatched=False; df itself is not modified. A row that satisfies several conditions is treated differently by the two modes. With drop_unmatched=True it is repeated once for every matching label (it keeps its original index value, so the index can contain duplicates) and the rows come out grouped by label, in the order of cond_dict. With drop_unmatched=False every row appears exactly once, in its original order, and carries the label of the last matching condition. An empty cond_dict raises ValueError when drop_unmatched=True.

Source code in Modeling_Tool/Core/utils.py

def bucket_by_cond(df: pd.DataFrame, cond_dict: dict, colname: str, 
                   drop_unmatched: bool = True, default=np.nan) -> pd.DataFrame:
    """
    Bucket the rows of a DataFrame by a dictionary of query conditions and tag each bucket with its label.

    Parameters
    ----------
    df : pandas.DataFrame
        Original data.
    cond_dict : dict
        Mapping of ``{label: query condition string}``.
    colname : str
        Name of the new label column.
    drop_unmatched : bool, default True
        If True, drop rows that match no condition (original behavior); if False, keep all rows.
    default : scalar, default np.nan
        Value assigned to unmatched rows (only used when ``drop_unmatched=False``).

    Returns
    -------
    pandas.DataFrame
        Data with the new label column.

    Notes
    -----
    The conditions are evaluated with ``DataFrame.query`` when ``drop_unmatched=True`` and with ``DataFrame.eval`` when
    ``drop_unmatched=False``; ``df`` itself is not modified. A row that satisfies several conditions is treated
    differently by the two modes. With ``drop_unmatched=True`` it is repeated once for every matching label (it keeps its
    original index value, so the index can contain duplicates) and the rows come out grouped by label, in the order of
    ``cond_dict``. With ``drop_unmatched=False`` every row appears exactly once, in its original order, and carries the
    label of the last matching condition. An empty ``cond_dict`` raises ``ValueError`` when ``drop_unmatched=True``.
    """
    if drop_unmatched:
        res_list = []
        for label, cond in cond_dict.items():
            sub = df.query(cond).copy()
            sub[colname] = label
            res_list.append(sub)
        return pd.concat(res_list)
    else:
        result = df.copy()
        result[colname] = default
        for label, cond in cond_dict.items():
            mask = result.eval(cond)
            result.loc[mask, colname] = label
        return result

cut2pieces

cut2pieces(varlist, n=4)

Split a list into several sub-lists.

Split the list into consecutive chunks of len(varlist) // n elements. The elements left over at the end are added to the last chunk, so that chunk is the longest, and the number of chunks is not always n (see Notes).

Parameters:

Name	Type	Description	Default
varlist	list	List to split.	required
n	int	Number of sub-lists wanted. It sets the chunk size len(varlist) // n and must not exceed len(varlist).	4

Returns:

Type	Description
list	The resulting sub-lists, in order (concatenating them gives varlist back).

Notes

The number of chunks is ceil(len(varlist) / (len(varlist) // n)) - 1, which is n only for some lengths: 10 items with n=4 give 4 chunks (sizes 2, 2, 2, 4), whereas 8 items with n=4 give only 3 chunks (sizes 2, 2, 4). The call fails when n is larger than len(varlist) (the chunk size is 0: ValueError) and when that formula gives fewer than 2 chunks, which is the case for n=1 and for n=2 with an even number of items (IndexError).

Examples:

>>> cut2pieces([1, 2, 3, 4, 5, 6, 7, 8], n=4)
[[1, 2], [3, 4], [5, 6, 7, 8]]
>>> cut2pieces(list(range(1, 11)), n=4)
[[1, 2], [3, 4], [5, 6], [7, 8, 9, 10]]

Source code in Modeling_Tool/Core/utils.py

def cut2pieces(varlist, n = 4):
    """
    Split a list into several sub-lists.

    Split the list into consecutive chunks of ``len(varlist) // n`` elements. The elements left over at the end are added
    to the last chunk, so that chunk is the longest, and the number of chunks is not always ``n`` (see Notes).

    Parameters
    ----------
    varlist : list
        List to split.
    n : int, default 4
        Number of sub-lists wanted. It sets the chunk size ``len(varlist) // n`` and must not exceed ``len(varlist)``.

    Returns
    -------
    list
        The resulting sub-lists, in order (concatenating them gives ``varlist`` back).

    Notes
    -----
    The number of chunks is ``ceil(len(varlist) / (len(varlist) // n)) - 1``, which is ``n`` only for some lengths: 10
    items with ``n=4`` give 4 chunks (sizes 2, 2, 2, 4), whereas 8 items with ``n=4`` give only 3 chunks (sizes 2, 2, 4).
    The call fails when ``n`` is larger than ``len(varlist)`` (the chunk size is 0: ``ValueError``) and when that formula
    gives fewer than 2 chunks, which is the case for ``n=1`` and for ``n=2`` with an even number of items (``IndexError``).

    Examples
    --------
    >>> cut2pieces([1, 2, 3, 4, 5, 6, 7, 8], n=4)
    [[1, 2], [3, 4], [5, 6, 7, 8]]
    >>> cut2pieces(list(range(1, 11)), n=4)
    [[1, 2], [3, 4], [5, 6], [7, 8, 9, 10]]
    """

    cut_point = np.floor(len(varlist) / n)
    cut_range = range(0, len(varlist), int(cut_point))
    cut_points = [x for x in cut_range]
    cut_points = cut_points[1: len(cut_points) - 1]

    cut_list = []

    i = 0
    while i <= len(cut_points):
        if i == 0:
            cut_list.append(varlist[:cut_points[i]])
        elif i == len(cut_points):
            cut_list.append(varlist[cut_points[i - 1]:])
        else: 
            cut_list.append(varlist[cut_points[i - 1]:cut_points[i]])
        i += 1

    return cut_list

check_colname_exist

check_colname_exist(data, colname)

Check whether a column name exists in a DataFrame.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
colname	str	Column name to check.	required

Returns:

Type	Description
bool	True if the column name exists, False otherwise.

Examples:

>>> df = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})
>>> check_colname_exist(df, 'a')
True

Source code in Modeling_Tool/Core/utils.py

def check_colname_exist(data, colname):
    """
    Check whether a column name exists in a DataFrame.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    colname : str
        Column name to check.

    Returns
    -------
    bool
        True if the column name exists, False otherwise.

    Examples
    --------
    >>> df = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})
    >>> check_colname_exist(df, 'a')
    True
    """

    return colname in data.columns

get_curr_abs_path

get_curr_abs_path(path)

Get the absolute path of a path relative to the current module's directory.

Parameters:

Name	Type	Description	Default
path	str	Relative path.	required

Returns:

Type	Description
str	Absolute path string.

Source code in Modeling_Tool/Core/utils.py

def get_curr_abs_path(path):
    """
    Get the absolute path of a path relative to the current module's directory.

    Parameters
    ----------
    path : str
        Relative path.

    Returns
    -------
    str
        Absolute path string.
    """
    return os.path.dirname(os.path.abspath(__file__)) + "/" + path

get_curr_datetime

get_curr_datetime(sep='')

Get the current date and time as a string.

Parameters:

Name	Type	Description	Default
sep	str	Separator between the date and the time.	''

Returns:

Type	Description
str	Formatted date-time string in the format YYYYMMDD{sep}HHMMSS.

Examples:

>>> get_curr_datetime()  # returns something like '20250330143624'
>>> get_curr_datetime('-')  # returns something like '20250330-143624'

Source code in Modeling_Tool/Core/utils.py

def get_curr_datetime(sep=''):
    """
    Get the current date and time as a string.

    Parameters
    ----------
    sep : str, default ''
        Separator between the date and the time.

    Returns
    -------
    str
        Formatted date-time string in the format YYYYMMDD{sep}HHMMSS.

    Examples
    --------
    >>> get_curr_datetime()  # returns something like '20250330143624'
    >>> get_curr_datetime('-')  # returns something like '20250330-143624'
    """
    import datetime as dt
    return dt.datetime.now().strftime(f"%Y%m%d{sep}%H%M%S")

get_buffer_date

get_buffer_date(start_date)

Get the date 4 weeks (28 days) before the start date.

Parameters:

Name	Type	Description	Default
start_date	str	Start date in the format 'YYYY-MM-DD'.	required

Returns:

Type	Description
str	Date 4 weeks before the start date, in the format 'YYYY-MM-DD'.

Examples:

>>> get_buffer_date('2025-03-30')
'2025-03-02'

Source code in Modeling_Tool/Core/utils.py

def get_buffer_date(start_date):
    """
    Get the date 4 weeks (28 days) before the start date.

    Parameters
    ----------
    start_date : str
        Start date in the format 'YYYY-MM-DD'.

    Returns
    -------
    str
        Date 4 weeks before the start date, in the format 'YYYY-MM-DD'.

    Examples
    --------
    >>> get_buffer_date('2025-03-30')
    '2025-03-02'
    """
    import datetime
    d = datetime.date(int(start_date[0:4]),int(start_date[5:7]),int(start_date[8:10]))
    res = (d + datetime.timedelta(weeks=-4)).strftime("%Y-%m-%d")
    return res

get_quarter

get_quarter(strDate)

Get the quarter from a date string.

Parameters:

Name	Type	Description	Default
strDate	str	Date string in the format 'YYYYMM' or 'YYYYMMDD'.	required

Returns:

Type	Description
int	Quarter (1-4).

Examples:

>>> get_quarter('202501')
1
>>> get_quarter('202506')
2

Source code in Modeling_Tool/Core/utils.py

def get_quarter(strDate):
    """
    Get the quarter from a date string.

    Parameters
    ----------
    strDate : str
        Date string in the format 'YYYYMM' or 'YYYYMMDD'.

    Returns
    -------
    int
        Quarter (1-4).

    Examples
    --------
    >>> get_quarter('202501')
    1
    >>> get_quarter('202506')
    2
    """
    return ((int(strDate[4:6])-1)//3) + 1

get_last_vintage

get_last_vintage()

Get the year-month string of the previous month.

Returns:

Type	Description
str	Year-month string of the previous month, in the format 'YYYYMM'.

Examples:

>>> get_last_vintage()  # returns '202502' if the current month is March 2025

Source code in Modeling_Tool/Core/utils.py

def get_last_vintage():
    """
    Get the year-month string of the previous month.

    Returns
    -------
    str
        Year-month string of the previous month, in the format 'YYYYMM'.

    Examples
    --------
    >>> get_last_vintage()  # returns '202502' if the current month is March 2025
    """
    import datetime
    todayDate = datetime.date.today()
    lastM = todayDate.replace(day=1) - datetime.timedelta(days=1)
    return lastM.strftime("%Y%m")

read_csv

read_csv(path, *args, **kwargs)

Read a CSV file and return a kDataFrame.

Parameters:

Name	Type	Description	Default
path	str	Path to the CSV file.	required
*args		Additional positional arguments passed to pandas.read_csv.	()
**kwargs		Additional keyword arguments passed to pandas.read_csv.	{}

Returns:

Type	Description
kDataFrame	kDataFrame containing the data.

Examples:

>>> df = read_csv('data.csv')

Source code in Modeling_Tool/Core/utils.py

def read_csv(path, *args, **kwargs):
    """
    Read a CSV file and return a kDataFrame.

    Parameters
    ----------
    path : str
        Path to the CSV file.
    *args
        Additional positional arguments passed to pandas.read_csv.
    **kwargs
        Additional keyword arguments passed to pandas.read_csv.

    Returns
    -------
    kDataFrame
        kDataFrame containing the data.

    Examples
    --------
    >>> df = read_csv('data.csv')
    """
    data = kDataFrame(pd.read_csv(path, *args, **kwargs))
    return data

df_to_h2oframe

df_to_h2oframe(data)

Convert a DataFrame to an H2OFrame.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame. An object that already is an h2o.H2OFrame is returned unchanged.	required

Returns:

Type	Description
H2OFrame	The H2OFrame object.

Notes

h2o is imported inside the function and must be installed.

Examples:

>>> hf = df_to_h2oframe(df)

Source code in Modeling_Tool/Core/utils.py

def df_to_h2oframe(data):
    """
    Convert a DataFrame to an H2OFrame.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame. An object that already is an ``h2o.H2OFrame`` is returned unchanged.

    Returns
    -------
    h2o.H2OFrame
        The H2OFrame object.

    Notes
    -----
    ``h2o`` is imported inside the function and must be installed.

    Examples
    --------
    >>> hf = df_to_h2oframe(df)
    """
    import h2o
    if isinstance(data, h2o.H2OFrame):
        return data
    else:
        return h2o.H2OFrame(data)

move_column

move_column(data, colname, idx, return_kDF=True, h2o_frame=False)

Move the given column to a specific position in a DataFrame.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame (an h2o.H2OFrame when h2o_frame=True).	required
colname	str	Name of the column to move.	required
idx	int	Target position (0-based) of the column, with list.insert semantics: 0 puts it first and a value at or beyond the number of columns puts it last.	required
return_kDF	bool	Whether to return a kDataFrame. Ignored (treated as False) when h2o_frame=True.	True
h2o_frame	bool	Whether the input is an H2OFrame.	False

Returns:

Type	Description
(DataFrame, kDataFrame or H2OFrame)	Data with the columns reordered (a new frame; data itself is not reordered).

Notes

h2o is imported unconditionally at the start of the function, so the call raises ModuleNotFoundError when h2o is not installed, even for pandas input.

Examples:

>>> df = pd.DataFrame({'a': [1, 2], 'b': [3, 4], 'c': [5, 6]})
>>> move_column(df, 'c', 0)  # move column 'c' to the first position

Source code in Modeling_Tool/Core/utils.py

def move_column(data, colname, idx, return_kDF = True, h2o_frame = False):
    """
    Move the given column to a specific position in a DataFrame.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame (an ``h2o.H2OFrame`` when ``h2o_frame=True``).
    colname : str
        Name of the column to move.
    idx : int
        Target position (0-based) of the column, with ``list.insert`` semantics: 0 puts it first and a value at or beyond
        the number of columns puts it last.
    return_kDF : bool, default True
        Whether to return a kDataFrame. Ignored (treated as False) when ``h2o_frame=True``.
    h2o_frame : bool, default False
        Whether the input is an H2OFrame.

    Returns
    -------
    pandas.DataFrame, kDataFrame or h2o.H2OFrame
        Data with the columns reordered (a new frame; ``data`` itself is not reordered).

    Notes
    -----
    ``h2o`` is imported unconditionally at the start of the function, so the call raises ``ModuleNotFoundError`` when h2o
    is not installed, even for pandas input.

    Examples
    --------
    >>> df = pd.DataFrame({'a': [1, 2], 'b': [3, 4], 'c': [5, 6]})
    >>> move_column(df, 'c', 0)  # move column 'c' to the first position
    """
    import h2o
    if h2o_frame:
        return_kDF = False
        colarray = data.columns 
    else:
        colarray = data.columns.tolist()
    colarray.remove(colname)
    colarray.insert(idx, colname)
    data = data[colarray]
    if return_kDF:
        return kDataFrame(data)
    return data

convert_to_vintage

convert_to_vintage(data, vintage_colname='VINTAGE', by='TRAN_TMS', return_kDF=True)

Generate a vintage column from a time column.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
vintage_colname	str	Name of the generated vintage column (a column with this name is overwritten).	'VINTAGE'
by	str	Name of the time column. The column is cast to string and the first YYYY-MM pattern in the text is used.	'TRAN_TMS'
return_kDF	bool	Whether to return a kDataFrame.	True

Returns:

Type	Description
DataFrame or kDataFrame	Data with the vintage column added.

Notes

The vintage column is added to the input data itself (in place), so the caller's frame gains the column as well as the returned one. The column has pandas string dtype and holds the vintage as YYYYMM text (for example '202503'); values without a YYYY-MM pattern (other separators, compact dates, missing values) become missing.

Examples:

>>> df = pd.DataFrame({'TRAN_TMS': ['2025-03-15 10:00:00', '2025-03-20 11:00:00']})
>>> convert_to_vintage(df)

Source code in Modeling_Tool/Core/utils.py

def convert_to_vintage(data, vintage_colname = 'VINTAGE', by = 'TRAN_TMS', return_kDF = True):
    """
    Generate a vintage column from a time column.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    vintage_colname : str, default 'VINTAGE'
        Name of the generated vintage column (a column with this name is overwritten).
    by : str, default 'TRAN_TMS'
        Name of the time column. The column is cast to string and the first ``YYYY-MM`` pattern in the text is used.
    return_kDF : bool, default True
        Whether to return a kDataFrame.

    Returns
    -------
    pandas.DataFrame or kDataFrame
        Data with the vintage column added.

    Notes
    -----
    The vintage column is added to the input ``data`` itself (in place), so the caller's frame gains the column as well as
    the returned one. The column has pandas ``string`` dtype and holds the vintage as ``YYYYMM`` text (for example
    ``'202503'``); values without a ``YYYY-MM`` pattern (other separators, compact dates, missing values) become missing.

    Examples
    --------
    >>> df = pd.DataFrame({'TRAN_TMS': ['2025-03-15 10:00:00', '2025-03-20 11:00:00']})
    >>> convert_to_vintage(df)
    """
    data[vintage_colname] = (
        data[by]
        .astype("string")
        .str.extract(r"(\d{4}-\d{2})", expand=False)
        .str.replace("-", "", regex=False)
    )

    if return_kDF:
        return kDataFrame(data)
    return data

col_filter_regex

col_filter_regex(data, regex='.*?of_co_at_12m', case_sensitive=True, h2o_frame=False, return_kDF=True)

Filter the columns of a DataFrame by matching the column names against a regular expression.

Parameters:

Name	Type	Description	Default
data	DataFrame or H2OFrame	Input data.	required
regex	str	Regular expression pattern, searched (re.search) anywhere in each column name.	".*?of_co_at_12m"
case_sensitive	bool	Whether matching is case-sensitive. Ignored when h2o_frame=True (the match is always case-sensitive there).	True
h2o_frame	bool	Whether the input is an H2OFrame. If True, return_kDF is ignored and an H2OFrame is returned.	False
return_kDF	bool	Whether to return a kDataFrame. Ignored (treated as False) when h2o_frame=True.	True

Returns:

Type	Description
(DataFrame, kDataFrame or H2OFrame)	Filtered data (only the matching columns, in their original order).

Notes

For pandas input the column names must be strings.

Examples:

>>> df = pd.DataFrame({'score_at_12m': [1, 2], 'other_col': [3, 4]})
>>> col_filter_regex(df, regex='score_at_12m')

Source code in Modeling_Tool/Core/utils.py

def col_filter_regex(data, regex = ".*?of_co_at_12m", case_sensitive = True, h2o_frame=False, return_kDF = True):
    """
    Filter the columns of a DataFrame by matching the column names against a regular expression.

    Parameters
    ----------
    data : pandas.DataFrame or h2o.H2OFrame
        Input data.
    regex : str, default ".*?of_co_at_12m"
        Regular expression pattern, searched (``re.search``) anywhere in each column name.
    case_sensitive : bool, default True
        Whether matching is case-sensitive. Ignored when ``h2o_frame=True`` (the match is always case-sensitive there).
    h2o_frame : bool, default False
        Whether the input is an H2OFrame. If True, ``return_kDF`` is ignored and an H2OFrame is returned.
    return_kDF : bool, default True
        Whether to return a kDataFrame. Ignored (treated as False) when ``h2o_frame=True``.

    Returns
    -------
    pandas.DataFrame, kDataFrame or h2o.H2OFrame
        Filtered data (only the matching columns, in their original order).

    Notes
    -----
    For pandas input the column names must be strings.

    Examples
    --------
    >>> df = pd.DataFrame({'score_at_12m': [1, 2], 'other_col': [3, 4]})
    >>> col_filter_regex(df, regex='score_at_12m')
    """
    if h2o_frame:
        return_kDF = False
        import re
        fltr = []
        for col in data.columns:
            if re.search(regex, col):
                fltr.append(col)
    else:
        fltr = data.columns[data.columns.str.contains(regex, regex = True, case = case_sensitive)]
    if return_kDF:
        return kDataFrame(data[fltr])
    return data[fltr]

row_filter_regex

row_filter_regex(data, col, regex, case_sensitive=True, as_index=False, return_kDF=True)

Filter the rows of a DataFrame by matching one column against a regular expression.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
col	str	Name of the column to filter on.	required
regex	str	Regular expression pattern, searched (re.search) anywhere in the text of each value.	required
case_sensitive	bool	Whether matching is case-sensitive.	True
as_index	bool	Whether to use the filter column as the index. It is only honored when return_kDF=False: with the default return_kDF=True the rows are returned with their original index.	False
return_kDF	bool	Whether to return a kDataFrame. When True, as_index has no effect.	True

Returns:

Type	Description
DataFrame or kDataFrame	Filtered data.

Notes

The values of col are cast to str before matching, so numbers are matched through their text and missing values through 'nan' or 'None'.

Examples:

>>> df = pd.DataFrame({'name': ['apple', 'banana', 'cherry'], 'value': [1, 2, 3]})
>>> row_filter_regex(df, 'name', 'a.*')

Source code in Modeling_Tool/Core/utils.py

def row_filter_regex(data, col, regex, case_sensitive = True,
                     as_index = False, return_kDF = True):
    """
    Filter the rows of a DataFrame by matching one column against a regular expression.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    col : str
        Name of the column to filter on.
    regex : str
        Regular expression pattern, searched (``re.search``) anywhere in the text of each value.
    case_sensitive : bool, default True
        Whether matching is case-sensitive.
    as_index : bool, default False
        Whether to use the filter column as the index. It is only honored when ``return_kDF=False``: with the default
        ``return_kDF=True`` the rows are returned with their original index.
    return_kDF : bool, default True
        Whether to return a kDataFrame. When True, ``as_index`` has no effect.

    Returns
    -------
    pandas.DataFrame or kDataFrame
        Filtered data.

    Notes
    -----
    The values of ``col`` are cast to ``str`` before matching, so numbers are matched through their text and missing values
    through ``'nan'`` or ``'None'``.

    Examples
    --------
    >>> df = pd.DataFrame({'name': ['apple', 'banana', 'cherry'], 'value': [1, 2, 3]})
    >>> row_filter_regex(df, 'name', 'a.*')
    """
    fltr = data[col].astype('str').str.contains(pat = regex, regex = True, case = case_sensitive)
    if return_kDF:
        return kDataFrame(data[fltr])
    if as_index:
        return data[fltr].set_index(col)
    return data[fltr]

convert_colnames

convert_colnames(data, how='lowercase', return_kDF=True)

Convert the column names of a DataFrame to a uniform case.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
how	str	Conversion method. One of 'lower'/'lowercase', 'upper'/'uppercase', 'cap'/'capitalize' (compared case-insensitively).	"lowercase"
return_kDF	bool	Whether to return a kDataFrame.	True

Returns:

Type	Description
DataFrame or kDataFrame	Data with the converted column names.

Notes

The columns of the input data are renamed in place, so the caller's frame changes as well as the returned one. The column names must be strings. how is not validated: any other value fails with UnboundLocalError.

Examples:

>>> df = pd.DataFrame({'NAME': [1], 'Age': [2]})
>>> convert_colnames(df, 'lower')

Source code in Modeling_Tool/Core/utils.py

def convert_colnames(data, how = "lowercase", return_kDF = True):
    """
    Convert the column names of a DataFrame to a uniform case.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    how : str, default "lowercase"
        Conversion method. One of 'lower'/'lowercase', 'upper'/'uppercase', 'cap'/'capitalize' (compared
        case-insensitively).
    return_kDF : bool, default True
        Whether to return a kDataFrame.

    Returns
    -------
    pandas.DataFrame or kDataFrame
        Data with the converted column names.

    Notes
    -----
    The columns of the input ``data`` are renamed in place, so the caller's frame changes as well as the returned one. The
    column names must be strings. ``how`` is not validated: any other value fails with ``UnboundLocalError``.

    Examples
    --------
    >>> df = pd.DataFrame({'NAME': [1], 'Age': [2]})
    >>> convert_colnames(df, 'lower')
    """
    cols = data.columns
    if how.lower() == "lower" or how.lower() == "lowercase":
        res = [name.lower() for name in cols]
    if how.lower() == "upper" or how.lower() == "uppercase":
        res = [name.upper() for name in cols]
    if how.lower() == "cap" or how.lower() == "capitalize":
        res = [name.capitalize() for name in cols]
    data.columns = res
    if return_kDF:
        return kDataFrame(data)
    return data

proc_freq

proc_freq(data, var: str, return_kDF=True) -> DataFrame

Compute frequencies and percentages, mimicking SAS PROC FREQ.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
var	str	Name of the column to tabulate.	required
return_kDF	bool	Whether to return a kDataFrame.	True

Returns:

Type	Description
DataFrame	Statistics table with the columns frequency, percent, cumFrequency and cumPercent. It is indexed by the distinct values of var (missing values form a row of their own) and sorted by that index, not by frequency. A kDataFrame when return_kDF=True.

Examples:

>>> df = pd.DataFrame({'category': ['A', 'B', 'A', 'C', 'A']})
>>> proc_freq(df, 'category')

Source code in Modeling_Tool/Core/utils.py

def proc_freq(data, var: str, return_kDF = True) -> pd.DataFrame:
    """
    Compute frequencies and percentages, mimicking SAS PROC FREQ.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    var : str
        Name of the column to tabulate.
    return_kDF : bool, default True
        Whether to return a kDataFrame.

    Returns
    -------
    pandas.DataFrame
        Statistics table with the columns frequency, percent, cumFrequency and cumPercent. It is indexed by the distinct
        values of ``var`` (missing values form a row of their own) and sorted by that index, not by frequency. A
        kDataFrame when ``return_kDF=True``.

    Examples
    --------
    >>> df = pd.DataFrame({'category': ['A', 'B', 'A', 'C', 'A']})
    >>> proc_freq(df, 'category')
    """

    f = data[var].value_counts(dropna = False)
    p = data[var].value_counts(dropna = False, normalize = True)
    df = pd.concat([f,p], axis = 1, keys = ['frequency', 'percent'])
    df = df.sort_index()
    df['cumFrequency'] = df['frequency'].cumsum()
    df['cumPercent'] = df['percent'].cumsum()
    if return_kDF:
        return kDataFrame(df)
    return df

quantile_rename_map

quantile_rename_map(columns)

Map the percentile labels that DataFrame.describe creates ('29%', '2.5%', '50%') to Q<percent>.

The label is read back from its text, so 0.29 gives 'Q29': computing int(0.29 * 100) gave 28 (the product is 28.999999999999996), which never matched the pandas label '29%', and a fractional percent kept the pandas label.

Parameters:

Name	Type	Description	Default
columns	iterable	Column labels of a describe result.	required

Returns:

Type	Description
dict	{label: "Q<percent>"} for every label that is a number followed by %; for example {'5%': 'Q5', '2.5%': 'Q2.5', '50%': 'Q50'}.

Source code in Modeling_Tool/Core/utils.py

def quantile_rename_map(columns):
    """Map the percentile labels that ``DataFrame.describe`` creates (``'29%'``, ``'2.5%'``, ``'50%'``) to ``Q<percent>``.

    The label is read back from its text, so ``0.29`` gives ``'Q29'``: computing ``int(0.29 * 100)`` gave 28 (the product is
    28.999999999999996), which never matched the pandas label ``'29%'``, and a fractional percent kept the pandas label.

    Parameters
    ----------
    columns : iterable
        Column labels of a ``describe`` result.

    Returns
    -------
    dict
        ``{label: "Q<percent>"}`` for every label that is a number followed by ``%``; for example ``{'5%': 'Q5',
        '2.5%': 'Q2.5', '50%': 'Q50'}``.
    """
    mapping = {}
    for column in columns:
        text = str(column)
        if text.endswith("%"):
            try:
                percent = float(text[:-1])
            except ValueError:
                continue
            mapping[column] = "Q" + format(percent, "g")
    return mapping

proc_means

proc_means(data, varlist=None, quantiles=[0.05, 0.15, 0.25, 0.5, 0.75, 0.95, 0.99])

Compute descriptive statistics, mimicking SAS PROC MEANS.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
varlist	list	List of columns to summarize; defaults to all columns.	None
quantiles	list	List of quantiles.	[0.05, 0.15, 0.25, 0.5, 0.75, 0.95, 0.99]

Returns:

Type	Description
DataFrame	Statistics table with one row per summarized column and the upper-case columns N (count of non-missing values), MEAN, STD, MIN, one Q<percent> column per requested quantile (Q5 for 0.05, Q50 for 0.5), MAX and MISSING_RATE (1 - N / number of rows of data).

Notes

Only numeric columns are summarized (the default of DataFrame.describe): non-numeric columns of varlist are silently left out of the result. Every quantile column is named Q<percent> (0.29 gives Q29, 0.075 gives Q7.5), and the median column, which describe always adds, is Q50.

Examples:

>>> df = pd.DataFrame({'a': [1, 2, 3, 4, 5], 'b': [10, 20, 30, 40, 50]})
>>> proc_means(df)

Source code in Modeling_Tool/Core/utils.py

def proc_means(data, varlist = None, quantiles = [0.05, 0.15, 0.25, 0.5, 0.75, 0.95, 0.99]):
    """
    Compute descriptive statistics, mimicking SAS PROC MEANS.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    varlist : list, optional
        List of columns to summarize; defaults to all columns.
    quantiles : list, default [0.05, 0.15, 0.25, 0.5, 0.75, 0.95, 0.99]
        List of quantiles.

    Returns
    -------
    pandas.DataFrame
        Statistics table with one row per summarized column and the upper-case columns ``N`` (count of non-missing
        values), ``MEAN``, ``STD``, ``MIN``, one ``Q<percent>`` column per requested quantile (``Q5`` for 0.05, ``Q50`` for
        0.5), ``MAX`` and ``MISSING_RATE`` (``1 - N / number of rows of data``).

    Notes
    -----
    Only numeric columns are summarized (the default of ``DataFrame.describe``): non-numeric columns of ``varlist`` are
    silently left out of the result. Every quantile column is named ``Q<percent>`` (``0.29`` gives ``Q29``, ``0.075``
    gives ``Q7.5``), and the median column, which ``describe`` always adds, is ``Q50``.

    Examples
    --------
    >>> df = pd.DataFrame({'a': [1, 2, 3, 4, 5], 'b': [10, 20, 30, 40, 50]})
    >>> proc_means(df)
    """

    if varlist is None:
        varlist = data.columns

    means = data[varlist].describe(percentiles = quantiles).T.rename(columns={"count":"n"})

    # Rename colnames.
    means.columns = [x.upper() for x in means.columns]
    means = means.rename(columns = quantile_rename_map(means.columns))

    # Compute Missing Rate
    means["MISSING_RATE"] = 1 - means["N"]/data.shape[0]
    return means

capping_score

capping_score(data, pb_score: str, multiplier=1, df_type: str = 'DataFrame')

Scale model scores and cap them at an upper limit.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame (an h2o.H2OFrame when df_type='h2o').	required
pb_score	str	Name of the score column.	required
multiplier	float	Score scaling multiplier.	1
df_type	str	Data type, 'DataFrame' or 'h2o' (compared case-insensitively). Any value other than 'h2o' selects the pandas branch.	'DataFrame'

Returns:

Type	Description
Series or H2OFrame	Processed scores: data[pb_score] * multiplier with every value above 0.9999999 replaced by 0.9999999.

Notes

Known limitation: as implemented, the pandas branch applies the cap with a scalar if on the whole column, so it raises ValueError ("The truth value of a Series is ambiguous") for any pandas input; only df_type='h2o' works.

Examples:

>>> capping_score(hf, 'score', multiplier=1, df_type='h2o')  # hf is an h2o.H2OFrame

Source code in Modeling_Tool/Core/utils.py

def capping_score(data, pb_score: str, multiplier = 1, df_type: str = 'DataFrame'):
    """
    Scale model scores and cap them at an upper limit.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame (an ``h2o.H2OFrame`` when ``df_type='h2o'``).
    pb_score : str
        Name of the score column.
    multiplier : float, default 1
        Score scaling multiplier.
    df_type : str, default 'DataFrame'
        Data type, 'DataFrame' or 'h2o' (compared case-insensitively). Any value other than 'h2o' selects the pandas
        branch.

    Returns
    -------
    pandas.Series or h2o.H2OFrame
        Processed scores: ``data[pb_score] * multiplier`` with every value above 0.9999999 replaced by 0.9999999.

    Notes
    -----
    Known limitation: as implemented, the pandas branch applies the cap with a scalar ``if`` on the whole column, so it
    raises ``ValueError`` ("The truth value of a Series is ambiguous") for any pandas input; only ``df_type='h2o'`` works.

    Examples
    --------
    >>> capping_score(hf, 'score', multiplier=1, df_type='h2o')  # hf is an h2o.H2OFrame
    """
    scores = data[pb_score] * multiplier
    cond = (scores > 0.9999999)

    if df_type.lower() == 'h2o':
        return cond.ifelse(0.9999999, scores)

    return (0.9999999 if scores > 0.9999999 else scores)

get_filenames

get_filenames(path: str, regex: str) -> [str]

Get the names of the files under a path that match a regular expression.

The folder is searched recursively, including its sub-folders.

Parameters:

Name	Type	Description	Default
path	str	Folder path.	required
regex	str	Regular expression pattern, searched (re.search) in each file name (not in the whole path).	required

Returns:

Type	Description
list	List of matching file names, in os.walk order. Only the file names are returned, without the sub-folder in which each file was found.

Examples:

>>> get_filenames('/path/to/files', '.*\.csv')
['file1.csv', 'file2.csv']

Source code in Modeling_Tool/Core/utils.py

def get_filenames(path: str, regex: str) -> [str]:
    """
    Get the names of the files under a path that match a regular expression.

    The folder is searched recursively, including its sub-folders.

    Parameters
    ----------
    path : str
        Folder path.
    regex : str
        Regular expression pattern, searched (``re.search``) in each file name (not in the whole path).

    Returns
    -------
    list
        List of matching file names, in ``os.walk`` order. Only the file names are returned, without the sub-folder in
        which each file was found.

    Examples
    --------
    >>> get_filenames('/path/to/files', '.*\\.csv')
    ['file1.csv', 'file2.csv']
    """
    import re
    outfiles = []
    for (Dirs, subdirs, files) in os.walk(path):
        for file in files:
            if re.search(regex, file):
                outfiles.append(file)
    return outfiles 

sas_to_csv_by_folder

sas_to_csv_by_folder(folder_path: str)

Convert all SAS datasets in a folder to CSV files.

Parameters:

Name	Type	Description	Default
folder_path	str	Path of the folder containing the SAS files. It must end with a path separator (for example '/data/sas/'), because the file names are appended to it as they are.	required

Returns:

Type	Description
int	Execution status code (0 means success).

Notes

Every file whose name contains sas7bdat is converted with sas_to_csv and written next to the source file, with sas7bdat replaced by csv in its path. Sub-folders are also searched, but only the file names are kept, so a file located in a sub-folder is looked up in folder_path and the conversion fails.

Examples:

>>> sas_to_csv_by_folder('/path/to/sas/files/')

Source code in Modeling_Tool/Core/utils.py

def sas_to_csv_by_folder(folder_path: str):
    """
    Convert all SAS datasets in a folder to CSV files.

    Parameters
    ----------
    folder_path : str
        Path of the folder containing the SAS files. It must end with a path separator (for example ``'/data/sas/'``),
        because the file names are appended to it as they are.

    Returns
    -------
    int
        Execution status code (0 means success).

    Notes
    -----
    Every file whose name contains ``sas7bdat`` is converted with ``sas_to_csv`` and written next to the source file, with
    ``sas7bdat`` replaced by ``csv`` in its path. Sub-folders are also searched, but only the file names are kept, so a
    file located in a sub-folder is looked up in ``folder_path`` and the conversion fails.

    Examples
    --------
    >>> sas_to_csv_by_folder('/path/to/sas/files/')
    """
    filenames = get_filenames(path = folder_path, regex = ".*?sas7bdat")
    sasfilepaths = [folder_path + file for file in filenames]
    from tqdm import tqdm
    with tqdm(total = len(sasfilepaths), position = 0, leave = True, file = sys.stdout) as pbar:
        for i in range(len(sasfilepaths)):
            saspath = sasfilepaths[i]
            logger.info(f"=> converting {filenames[i]}...")
            csvpath = sasfilepaths[i].replace("sas7bdat", "csv")
            sas_to_csv(saspath, csvpath)
            pbar.update()
    return 0

read_attr_list

read_attr_list(path: str = 'pe_attr_list.txt', lower=False)

Read an attribute list file (one attribute per line).

Parameters:

Name	Type	Description	Default
path	str	File path.	"pe_attr_list.txt"
lower	bool	Whether to convert the attributes to lower case. With the default False they are returned in upper case.	False

Returns:

Type	Description
list	List of attributes, one per line of the file, with surrounding whitespace removed and converted to upper case (or to lower case when lower=True). Blank lines are kept as empty strings.

Examples:

>>> read_attr_list('vars.txt', lower=True)

Source code in Modeling_Tool/Core/utils.py

def read_attr_list(path: str = "pe_attr_list.txt", lower = False):
    """
    Read an attribute list file (one attribute per line).

    Parameters
    ----------
    path : str, default "pe_attr_list.txt"
        File path.
    lower : bool, default False
        Whether to convert the attributes to lower case. With the default False they are returned in upper case.

    Returns
    -------
    list
        List of attributes, one per line of the file, with surrounding whitespace removed and converted to upper case (or
        to lower case when ``lower=True``). Blank lines are kept as empty strings.

    Examples
    --------
    >>> read_attr_list('vars.txt', lower=True)
    """
    with open(path) as f:
        lines = f.readlines()

    ls = []
    for line in lines:
        ls.append(line.strip().upper())
    f.close()
    if lower:
        return [x.lower() for x in ls]
    return ls

write_attr_list

write_attr_list(var_list: list, path: str = '_vls_results.txt', sep='\n', quote='double')

Write a list of variables to a file.

Each variable is converted with str and written to the file followed by sep, so the file also ends with a separator. An existing file is overwritten.

Parameters:

Name	Type	Description	Default
var_list	list	List of variables to write.	required
path	str	Output file path.	"_vls_results.txt"
sep	str	Separator written after every variable, the last one included. The default is the newline character.	"\n"
quote	str	Quote type: 'double', 'single' or 'none'. Any other value is handled like 'none' (no quotes).	'double'

Returns:

Type	Description
None

Examples:

>>> write_attr_list(['var1', 'var2'], 'output.txt', quote='single')

Source code in Modeling_Tool/Core/utils.py

def write_attr_list(var_list: list, path: str = "_vls_results.txt", sep="\n", quote='double'):
    """
    Write a list of variables to a file.

    Each variable is converted with ``str`` and written to the file followed by ``sep``, so the file also ends with a
    separator. An existing file is overwritten.

    Parameters
    ----------
    var_list : list
        List of variables to write.
    path : str, default "_vls_results.txt"
        Output file path.
    sep : str, default "\\n"
        Separator written after every variable, the last one included. The default is the newline character.
    quote : str, default 'double'
        Quote type: 'double', 'single' or 'none'. Any other value is handled like 'none' (no quotes).

    Returns
    -------
    None

    Examples
    --------
    >>> write_attr_list(['var1', 'var2'], 'output.txt', quote='single')
    """
    with open(path, "w+") as f:
        for v in var_list:
            if quote == 'double':
                f.writelines('"'+str(v)+'"'+sep)
            elif quote == 'single':
                f.writelines("'"+str(v)+"'"+sep)
            else:
                f.writelines(str(v)+sep)
        f.close()
    return None

list_filter_regex

list_filter_regex(ls, regex)

Filter list elements by a regular expression.

Parameters:

Name	Type	Description	Default
ls	list	Input list (of strings).	required
regex	str	Regular expression pattern, searched (re.search) anywhere in each element.	required

Returns:

Type	Description
list	List of matching elements.

Examples:

>>> list_filter_regex(['abc', 'def', 'abf'], 'ab.*')
['abc', 'abf']

Source code in Modeling_Tool/Core/utils.py

def list_filter_regex(ls, regex):
    """
    Filter list elements by a regular expression.

    Parameters
    ----------
    ls : list
        Input list (of strings).
    regex : str
        Regular expression pattern, searched (``re.search``) anywhere in each element.

    Returns
    -------
    list
        List of matching elements.

    Examples
    --------
    >>> list_filter_regex(['abc', 'def', 'abf'], 'ab.*')
    ['abc', 'abf']
    """
    import re
    ret = []
    for elem in ls:
        if re.search(regex, elem):
            ret.append(elem)
    return ret

list_to_h2oFrame

list_to_h2oFrame(val: str or float or int, length: int)

Convert a value to an H2O Frame of the given length.

Parameters:

Name	Type	Description	Default
val	str or float or int	Value to repeat.	required
length	int	Length of the frame.	required

Returns:

Type	Description
H2OFrame	H2OFrame containing the repeated value.

Notes

h2o is imported inside the function and must be installed.

Examples:

>>> list_to_h2oFrame(5, 10)

Source code in Modeling_Tool/Core/utils.py

def list_to_h2oFrame(val: str or float or int, length: int):
    """
    Convert a value to an H2O Frame of the given length.

    Parameters
    ----------
    val : str or float or int
        Value to repeat.
    length : int
        Length of the frame.

    Returns
    -------
    h2o.H2OFrame
        H2OFrame containing the repeated value.

    Notes
    -----
    ``h2o`` is imported inside the function and must be installed.

    Examples
    --------
    >>> list_to_h2oFrame(5, 10)
    """
    """ convert list to H2O Frame """
    import h2o
    return h2o.H2OFrame([val] * length)

odds_score

odds_score(pb_score, event_ratio=15, margin_point=20, score_point=500)

Compute the odds score from a probability score.

Used to convert a probability into a credit score scale. The score is score_point - margin_point / ln(2) * (ln(event_ratio) + ln(pb_score / (1 - pb_score))).

Parameters:

Name	Type	Description	Default
pb_score	float or array - like	Predicted probability of the event, strictly between 0 and 1 (a scalar, a numpy array or a pandas Series).	required
event_ratio	float	Event ratio: the odds of non-event to event at which the score equals score_point. With the default 15 a probability of 1/16 scores exactly score_point.	15
margin_point	float	Score point difference: the points subtracted from the score every time the odds pb_score / (1 - pb_score) double.	20
score_point	float	Base score point.	500

Returns:

Type	Description
float or array - like	Odds score, of the same kind as pb_score. A higher probability gives a lower score.

Examples:

>>> odds_score(0.03, event_ratio=15, margin_point=20, score_point=500)
522.16...

Source code in Modeling_Tool/Core/utils.py

def odds_score(pb_score, event_ratio = 15, margin_point = 20, score_point = 500):
    """
    Compute the odds score from a probability score.

    Used to convert a probability into a credit score scale. The score is
    ``score_point - margin_point / ln(2) * (ln(event_ratio) + ln(pb_score / (1 - pb_score)))``.

    Parameters
    ----------
    pb_score : float or array-like
        Predicted probability of the event, strictly between 0 and 1 (a scalar, a numpy array or a pandas Series).
    event_ratio : float, default 15
        Event ratio: the odds of non-event to event at which the score equals ``score_point``. With the default 15 a
        probability of 1/16 scores exactly ``score_point``.
    margin_point : float, default 20
        Score point difference: the points subtracted from the score every time the odds ``pb_score / (1 - pb_score)``
        double.
    score_point : float, default 500
        Base score point.

    Returns
    -------
    float or array-like
        Odds score, of the same kind as ``pb_score``. A higher probability gives a lower score.

    Examples
    --------
    >>> odds_score(0.03, event_ratio=15, margin_point=20, score_point=500)
    522.16...
    """
    a = (margin_point / np.log(2))
    b = (np.log(event_ratio) + np.log(pb_score/(1 - pb_score)))
    return (score_point - a * b)

last_Month_Vintage

last_Month_Vintage(year: int, month: int, day: int) -> str

Get the year-month of the previous month as an integer in the format YYYYMM.

Parameters:

Name	Type	Description	Default
year	int	Year.	required
month	int	Month.	required
day	int	Day. It only has to be a valid day of month; it does not change the result.	required

Returns:

Type	Description
int	Previous month of the date year-month-day as an integer in the format YYYYMM (for example 202502). The -> str annotation of the function is inaccurate: an int is returned, not a string.

Examples:

>>> last_Month_Vintage(2025, 3, 15)
202502

Source code in Modeling_Tool/Core/utils.py

def last_Month_Vintage(year: int, month: int, day: int) -> str:
    """
    Get the year-month of the previous month as an integer in the format YYYYMM.

    Parameters
    ----------
    year : int
        Year.
    month : int
        Month.
    day : int
        Day. It only has to be a valid day of ``month``; it does not change the result.

    Returns
    -------
    int
        Previous month of the date ``year-month-day`` as an integer in the format YYYYMM (for example ``202502``). The
        ``-> str`` annotation of the function is inaccurate: an ``int`` is returned, not a string.

    Examples
    --------
    >>> last_Month_Vintage(2025, 3, 15)
    202502
    """
    lastDate = dt(year, month, day) + rd(months=-1)
    yr = str(lastDate.year)
    mth = "0" + str(lastDate.month) if len([char for char in str(lastDate.month)]) == 1 else str(lastDate.month)
    lastMthVtge = yr + mth
    return int(lastMthVtge)

read_sas_file

read_sas_file(file_path_name='')

Read a SAS dataset file.

The SAS file is read with latin-1 encoding, which is the default encoding of SAS Studio and SAS Grid.

Parameters:

Name	Type	Description	Default
file_path_name	str	Path to the SAS file (a .sas7bdat file: the format is fixed to sas7bdat). The empty default is not a usable path, so always pass one.	''

Returns:

Type	Description
kDataFrame	kDataFrame containing the data.

Examples:

>>> df = read_sas_file('data.sas7bdat')

Source code in Modeling_Tool/Core/utils.py

def read_sas_file(file_path_name=''):
    """
    Read a SAS dataset file.

    The SAS file is read with latin-1 encoding, which is the default encoding of SAS Studio and SAS Grid.

    Parameters
    ----------
    file_path_name : str, default ''
        Path to the SAS file (a ``.sas7bdat`` file: the format is fixed to ``sas7bdat``). The empty default is not a
        usable path, so always pass one.

    Returns
    -------
    kDataFrame
        kDataFrame containing the data.

    Examples
    --------
    >>> df = read_sas_file('data.sas7bdat')
    """
    df = pd.read_sas(file_path_name,
        format = 'sas7bdat', encoding="latin-1")
    return kDataFrame(df)

sas_to_csv

sas_to_csv(fileNameWithPath, outputFileNameWithPath, timecounter=True)

Convert a SAS dataset to a CSV file.

Parameters:

Name	Type	Description	Default
fileNameWithPath	str	Path to the input SAS file.	required
outputFileNameWithPath	str	Path to the output CSV file (written without the index column).	required
timecounter	bool	Whether to report the execution time (in minutes) through the logger at INFO level.	True

Returns:

Type	Description
int	Execution status code (0 means success).

Examples:

>>> sas_to_csv('input.sas7bdat', 'output.csv')
Completed! It took 0.1234 minutes to run.
0

Source code in Modeling_Tool/Core/utils.py

def sas_to_csv(fileNameWithPath, outputFileNameWithPath, timecounter = True):
    """
    Convert a SAS dataset to a CSV file.

    Parameters
    ----------
    fileNameWithPath : str
        Path to the input SAS file.
    outputFileNameWithPath : str
        Path to the output CSV file (written without the index column).
    timecounter : bool, default True
        Whether to report the execution time (in minutes) through the logger at INFO level.

    Returns
    -------
    int
        Execution status code (0 means success).

    Examples
    --------
    >>> sas_to_csv('input.sas7bdat', 'output.csv')
    Completed! It took 0.1234 minutes to run.
    0
    """
    import time as t

    t0 = t.time()
    df = read_sas_file(fileNameWithPath)
    df.to_csv(outputFileNameWithPath, index = False)
    t1 = t.time()
    if timecounter:
        logger.info(f"Completed! It took {round((t1 - t0)/60, 4)} minutes to run.")
    return 0

merge_all_data

merge_all_data(*args, on='APPLICATION_ID', how='left', return_kDF=True)

Merge multiple datasets.

Parameters:

Name	Type	Description	Default
*args		The DataFrames to merge, passed as positional arguments. They are merged from left to right; at least one is required.	()
on	str	Name of the join key column. It must exist in every DataFrame.	"APPLICATION_ID"
how	str	Join type: 'left', 'right', 'inner' or 'outer'.	"left"
return_kDF	bool	Whether to return a kDataFrame.	True

Returns:

Type	Description
DataFrame or kDataFrame	Merged data.

Notes

In the i-th merge (counting from 0) the non-key columns that exist on both sides get the suffixes _merge{i} (left) and _merge{i+1} (right). With a single DataFrame nothing is merged and that frame is returned; with no DataFrame the call fails with IndexError.

Examples:

>>> df1 = pd.DataFrame({'id': [1, 2], 'a': [3, 4]})
>>> df2 = pd.DataFrame({'id': [1, 2], 'b': [5, 6]})
>>> merge_all_data(df1, df2, on='id')

Source code in Modeling_Tool/Core/utils.py

def merge_all_data(*args, on = "APPLICATION_ID", how = "left", return_kDF = True):
    """
    Merge multiple datasets.

    Parameters
    ----------
    *args
        The DataFrames to merge, passed as positional arguments. They are merged from left to right; at least one is
        required.
    on : str, default "APPLICATION_ID"
        Name of the join key column. It must exist in every DataFrame.
    how : str, default "left"
        Join type: 'left', 'right', 'inner' or 'outer'.
    return_kDF : bool, default True
        Whether to return a kDataFrame.

    Returns
    -------
    pandas.DataFrame or kDataFrame
        Merged data.

    Notes
    -----
    In the i-th merge (counting from 0) the non-key columns that exist on both sides get the suffixes ``_merge{i}`` (left)
    and ``_merge{i+1}`` (right). With a single DataFrame nothing is merged and that frame is returned; with no DataFrame
    the call fails with ``IndexError``.

    Examples
    --------
    >>> df1 = pd.DataFrame({'id': [1, 2], 'a': [3, 4]})
    >>> df2 = pd.DataFrame({'id': [1, 2], 'b': [5, 6]})
    >>> merge_all_data(df1, df2, on='id')
    """
    argList = list(args)
    data = argList[0]
    for i in range(len(argList)-1):
        data = pd.merge(data, argList[i+1], on=on, how = how, suffixes=(f'_merge{i}', f'_merge{i+1}'))
    if return_kDF:
        return kDataFrame(data)
    return data

get_valid_vintages

get_valid_vintages(sVintage, eVintage)

Get the list of valid vintages within the given range.

Parameters:

Name	Type	Description	Default
sVintage	int	Start vintage (format YYYYMM).	required
eVintage	int	End vintage (format YYYYMM).	required

Returns:

Type	Description
list	List of valid vintages: the integers from sVintage to eVintage, both included, whose month part is 01 to 12.

Examples:

>>> get_valid_vintages(202001, 202503)
[202001, 202002, ..., 202012, 202101, ..., 202503]

Source code in Modeling_Tool/Core/utils.py

def get_valid_vintages(sVintage, eVintage):
    """
    Get the list of valid vintages within the given range.

    Parameters
    ----------
    sVintage : int
        Start vintage (format YYYYMM).
    eVintage : int
        End vintage (format YYYYMM).

    Returns
    -------
    list
        List of valid vintages: the integers from ``sVintage`` to ``eVintage``, both included, whose month part is 01 to 12.

    Examples
    --------
    >>> get_valid_vintages(202001, 202503)
    [202001, 202002, ..., 202012, 202101, ..., 202503]
    """
    vintages = []
    for vintage in range(sVintage, eVintage + 1):
        if (vintage % 100) > 0 and (vintage % 100) < 13:
            vintages.append(vintage)
    return vintages

set_non_number_str

set_non_number_str(h2o_tbl_path)

Import a file as an H2OFrame and set all non-numeric columns to string type.

Parameters:

Name	Type	Description	Default
h2o_tbl_path	str	Path to the file to import (anything accepted by h2o.import_file, for example a CSV file).	required

Returns:

Type	Description
H2OFrame	The processed H2OFrame: columns detected as int, enum, string or real keep their type and every other column is read as string.

Notes

The function calls h2o.init(min_mem_size='100G') (it connects to a running H2O cluster or starts a local one that asks for at least 100 GB of memory) and imports the file twice: first to detect the column types, then again with the final types. h2o must be installed.

Examples:

>>> hf = set_non_number_str('/path/to/file.csv')

Source code in Modeling_Tool/Core/utils.py

def set_non_number_str(h2o_tbl_path):
    """
    Import a file as an H2OFrame and set all non-numeric columns to string type.

    Parameters
    ----------
    h2o_tbl_path : str
        Path to the file to import (anything accepted by ``h2o.import_file``, for example a CSV file).

    Returns
    -------
    h2o.H2OFrame
        The processed H2OFrame: columns detected as ``int``, ``enum``, ``string`` or ``real`` keep their type and every
        other column is read as ``string``.

    Notes
    -----
    The function calls ``h2o.init(min_mem_size='100G')`` (it connects to a running H2O cluster or starts a local one that
    asks for at least 100 GB of memory) and imports the file twice: first to detect the column types, then again with the
    final types. ``h2o`` must be installed.

    Examples
    --------
    >>> hf = set_non_number_str('/path/to/file.csv')
    """
    """ Import file as H2O Frame and set all non-numeric columns to string type. """
    import h2o
    h2o.init(min_mem_size='100G')
    df = h2o.import_file(h2o_tbl_path)
    orig_types = list(df.types.values())
    fnl_types = ['string' if ((tp !='int') and (tp != 'enum') and (tp != 'string') and (tp != 'real')) else str(tp) for tp in orig_types]
    fnl_df = h2o.import_file(h2o_tbl_path, col_types = fnl_types)
    return fnl_df

list_to_SQL

list_to_SQL(ls, excl=[], prefix='', wquote=False)

Convert a list to a SQL-formatted string.

Parameters:

Name	Type	Description	Default
ls	list	Input list. Without wquote the elements must be strings.	required
excl	list	List of elements to exclude.	[]
prefix	str	Column-name prefix (such as a table alias), written before every element as prefix.element.	''
wquote	bool	Whether to wrap each element in single quotes.	False

Returns:

Type	Description
str	SQL-formatted string: the elements joined by commas without spaces.

Notes

With wquote=True the elements are quoted before they are compared with excl, so excl must then contain the quoted text ("'b'") to exclude anything. The comma after an element depends only on its position in ls, so when the last element of ls is excluded the result ends with a trailing comma (list_to_SQL(['a', 'b'], excl=['b']) returns 'a,').

Examples:

>>> list_to_SQL(['col1', 'col2', 'col3'], prefix='t')
't.col1,t.col2,t.col3'

Source code in Modeling_Tool/Core/utils.py

def list_to_SQL(ls, excl=[], prefix = '', wquote=False):
    """
    Convert a list to a SQL-formatted string.

    Parameters
    ----------
    ls : list
        Input list. Without ``wquote`` the elements must be strings.
    excl : list, default []
        List of elements to exclude.
    prefix : str, default ''
        Column-name prefix (such as a table alias), written before every element as ``prefix.element``.
    wquote : bool, default False
        Whether to wrap each element in single quotes.

    Returns
    -------
    str
        SQL-formatted string: the elements joined by commas without spaces.

    Notes
    -----
    With ``wquote=True`` the elements are quoted before they are compared with ``excl``, so ``excl`` must then contain the
    quoted text (``"'b'"``) to exclude anything. The comma after an element depends only on its position in ``ls``, so when
    the last element of ``ls`` is excluded the result ends with a trailing comma (``list_to_SQL(['a', 'b'], excl=['b'])``
    returns ``'a,'``).

    Examples
    --------
    >>> list_to_SQL(['col1', 'col2', 'col3'], prefix='t')
    't.col1,t.col2,t.col3'
    """
    sqlFmt = ""

    for i, var in enumerate(ls):
        if wquote:
            var = f"'{var}'"
        if var in excl:
            continue
        if i != len(ls) - 1:
            if prefix == '' or prefix is None:
                sqlFmt += var + ","
            else:
                sqlFmt += prefix + "." + var + ","
        else:
            if prefix == '' or prefix is None:
                sqlFmt += var
            else:
                sqlFmt += prefix + "." + var
    return sqlFmt

bool_to_str

bool_to_str(data)

Convert the boolean columns of a DataFrame to string type.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required

Returns:

Type	Description
DataFrame	Converted data: a copy in which every boolean column holds the strings 'True' / 'False'. data itself is not modified.

Examples:

>>> df = pd.DataFrame({'a': [True, False], 'b': [1, 2]})
>>> bool_to_str(df)

Source code in Modeling_Tool/Core/utils.py

def bool_to_str(data):
    """
    Convert the boolean columns of a DataFrame to string type.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.

    Returns
    -------
    pandas.DataFrame
        Converted data: a copy in which every boolean column holds the strings ``'True'`` / ``'False'``. ``data`` itself is
        not modified.

    Examples
    --------
    >>> df = pd.DataFrame({'a': [True, False], 'b': [1, 2]})
    >>> bool_to_str(df)
    """
    dfc = data.copy()
    bool_cols = [col for col in data.columns if pd.api.types.is_bool_dtype(data[col])]
    if bool_cols:
        dfc[bool_cols] = dfc[bool_cols].astype(str)
    return dfc

get_dtypes_file

get_dtypes_file(data, outputFile=None, ck_format=False)

Get the data type of each column of a DataFrame.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
outputFile	str	Output file path. If given, the table is also written there as CSV, without the index and without a header row.	None
ck_format	bool	Reserved for a ClickHouse dtype mapping that SMF does not ship: ck_format=True raises NotImplementedError (before anything is written), so leave it at False.	False

Returns:

Type	Description
DataFrame	DataFrame with the column names (column colname) and data types (column dtype, as strings such as 'int64').

Raises:

Type	Description
NotImplementedError	If ck_format=True.

Notes

Boolean columns are converted to strings first (see bool_to_str), so their type is reported as object.

Examples:

>>> df = pd.DataFrame({'a': [1], 'b': ['x'], 'c': [1.5]})
>>> get_dtypes_file(df)

Source code in Modeling_Tool/Core/utils.py

def get_dtypes_file(data, outputFile = None, ck_format=False):
    """
    Get the data type of each column of a DataFrame.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    outputFile : str, optional
        Output file path. If given, the table is also written there as CSV, without the index and without a header row.
    ck_format : bool, default False
        Reserved for a ClickHouse dtype mapping that SMF does not ship: ``ck_format=True`` raises
        ``NotImplementedError`` (before anything is written), so leave it at False.

    Returns
    -------
    pandas.DataFrame
        DataFrame with the column names (column ``colname``) and data types (column ``dtype``, as strings such as
        ``'int64'``).

    Raises
    ------
    NotImplementedError
        If ``ck_format=True``.

    Notes
    -----
    Boolean columns are converted to strings first (see ``bool_to_str``), so their type is reported as ``object``.

    Examples
    --------
    >>> df = pd.DataFrame({'a': [1], 'b': ['x'], 'c': [1.5]})
    >>> get_dtypes_file(df)
    """
    df = bool_to_str(data)
    res = pd.DataFrame(df.dtypes)
    res = res.reset_index()
    res.columns = ["colname", "dtype"]
    if ck_format:
        raise NotImplementedError(
            "ck_format=True needs a ClickHouse dtype mapping, which SMF does not ship. "
            "Use the default ck_format=False."
        )
    res['dtype'] = res['dtype'].astype(str).str.strip()
    if outputFile is not None:
        res.to_csv(outputFile, index = False, header=False)
    return res

add_path_suffix

add_path_suffix(file, suffix='_cut')

Add a suffix to a file path (before the file extension).

Parameters:

Name	Type	Description	Default
file	str	File path, with / as separator.	required
suffix	str	Suffix to add.	"_cut"

Returns:

Type	Description
str	File path with the suffix added.

Notes

The file name is split at its dots and only the first two parts are used, so 'a.tar.gz' becomes 'a_cut.tar' and a name without a dot fails with IndexError. A bare file name (no /) comes back with a leading / ('file.csv' gives '/file_cut.csv').

Examples:

>>> add_path_suffix('/path/to/file.csv', '_processed')
'/path/to/file_processed.csv'

Source code in Modeling_Tool/Core/utils.py

def add_path_suffix(file, suffix = "_cut"):
    """
    Add a suffix to a file path (before the file extension).

    Parameters
    ----------
    file : str
        File path, with ``/`` as separator.
    suffix : str, default "_cut"
        Suffix to add.

    Returns
    -------
    str
        File path with the suffix added.

    Notes
    -----
    The file name is split at its dots and only the first two parts are used, so ``'a.tar.gz'`` becomes ``'a_cut.tar'``
    and a name without a dot fails with ``IndexError``. A bare file name (no ``/``) comes back with a leading ``/``
    (``'file.csv'`` gives ``'/file_cut.csv'``).

    Examples
    --------
    >>> add_path_suffix('/path/to/file.csv', '_processed')
    '/path/to/file_processed.csv'
    """
    whole_path = file.split("/")
    path = [item for item in whole_path if item != whole_path[-1]]
    file = whole_path[-1].split(".")
    filename = file[0]
    ext = file[1]
    res = "/".join(path)+"/"+filename+suffix+"."+ext
    return res

h2o_apply_regex

h2o_apply_regex(data, colname, func)

Apply a regular-expression transformation function to a column of an H2O Frame.

Parameters:

Name	Type	Description	Default
data	H2OFrame	Input data.	required
colname	str	Column name. It must be a string.	required
func	callable	Function to apply to every element of the column.	required

Returns:

Type	Description
H2OFrame	The transformed H2OFrame: a new frame with the single column colname, which holds the transformed values (the other columns of data are not carried over).

Notes

The column is downloaded as a pandas Series, func is applied there element by element and the result is uploaded again, so the column must fit in memory. A colname that is not a string fails with UnboundLocalError. h2o is imported inside the function and must be installed.

Examples:

>>> h2o_apply_regex(hf, 'name', lambda x: x.upper())

Source code in Modeling_Tool/Core/utils.py

def h2o_apply_regex(data, colname, func):
    """
    Apply a regular-expression transformation function to a column of an H2O Frame.

    Parameters
    ----------
    data : h2o.H2OFrame
        Input data.
    colname : str
        Column name. It must be a string.
    func : callable
        Function to apply to every element of the column.

    Returns
    -------
    h2o.H2OFrame
        The transformed H2OFrame: a new frame with the single column ``colname``, which holds the transformed values (the
        other columns of ``data`` are not carried over).

    Notes
    -----
    The column is downloaded as a pandas Series, ``func`` is applied there element by element and the result is uploaded
    again, so the column must fit in memory. A ``colname`` that is not a string fails with ``UnboundLocalError``. ``h2o``
    is imported inside the function and must be installed.

    Examples
    --------
    >>> h2o_apply_regex(hf, 'name', lambda x: x.upper())
    """
    """ Apply lambda function to h2o frame. """
    import h2o  # optional dependency, only needed by this helper

    if isinstance(colname, str):
        fnl_res = h2o.H2OFrame(data[colname].as_data_frame()[colname].apply(func).tolist())
        fnl_res = fnl_res.rename({'C1':colname})
    return fnl_res

get_summary_rpt

get_summary_rpt(means_rpt, iv_psi_rpt, corr_rpt)

Merge reports into a feature summary report.

Combine the Means report, the IV/PSI report and the correlation report into one comprehensive report.

Parameters:

Name	Type	Description	Default
means_rpt	DataFrame	Means statistics report, indexed by variable name (for example the output of proc_means).	required
iv_psi_rpt	DataFrame	IV/PSI report. It needs a Var_Name column, which is used as its index.	required
corr_rpt	DataFrame	Correlation report. It needs a Var_Name column, which is used as its index.	required

Returns:

Type	Description
DataFrame	Merged summary report: the three reports joined on the variable name (an inner join, so a variable missing from any report is dropped), with all column names converted to upper case.

Examples:

>>> summary = get_summary_rpt(means, iv_psi, corr)

Source code in Modeling_Tool/Core/utils.py

def get_summary_rpt(means_rpt, iv_psi_rpt, corr_rpt):
    """
    Merge reports into a feature summary report.

    Combine the Means report, the IV/PSI report and the correlation report into one comprehensive report.

    Parameters
    ----------
    means_rpt : pandas.DataFrame
        Means statistics report, indexed by variable name (for example the output of ``proc_means``).
    iv_psi_rpt : pandas.DataFrame
        IV/PSI report. It needs a ``Var_Name`` column, which is used as its index.
    corr_rpt : pandas.DataFrame
        Correlation report. It needs a ``Var_Name`` column, which is used as its index.

    Returns
    -------
    pandas.DataFrame
        Merged summary report: the three reports joined on the variable name (an inner join, so a variable missing from
        any report is dropped), with all column names converted to upper case.

    Examples
    --------
    >>> summary = get_summary_rpt(means, iv_psi, corr)
    """
    iv_psi_rpt = iv_psi_rpt.set_index("Var_Name")
    corr_rpt = corr_rpt.set_index("Var_Name")

    fnl_rpt = means_rpt\
    .merge(iv_psi_rpt, left_index = True, right_index = True)\
    .merge(corr_rpt, left_index = True, right_index = True)

    fnl_rpt.columns = [x.upper() for x in fnl_rpt.columns]
    return fnl_rpt

flatten_json_attr

flatten_json_attr(data, jsonColname='data')

Flatten a column of JSON-format model attributes.

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the JSON column.	required
jsonColname	str	Name of the JSON column. Every cell must be a string that ast.literal_eval can evaluate to a dict.	"data"

Returns:

Type	Description
DataFrame	The flattened DataFrame: the columns of data other than the column named data, followed by the flattened keys (nested keys are joined with a dot, such as b.c).

Notes

The cells are parsed with ast.literal_eval, which means Python literal syntax: JSON true, false and null are not accepted. The column dropped from the output is the one literally named 'data', whatever jsonColname is, so with another jsonColname the JSON column itself stays in the result. The flattened keys are joined side by side (pd.concat(axis=1)) with a new 0 to n-1 index, so data should have the default index; with any other index the rows no longer line up (reset the index first).

Examples:

>>> df = pd.DataFrame({'id': [1], 'data': ['{"key1":"val1"}']})
>>> flatten_json_attr(df)

Source code in Modeling_Tool/Core/utils.py

def flatten_json_attr(data, jsonColname= "data"):
    """
    Flatten a column of JSON-format model attributes.

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the JSON column.
    jsonColname : str, default "data"
        Name of the JSON column. Every cell must be a string that ``ast.literal_eval`` can evaluate to a dict.

    Returns
    -------
    pandas.DataFrame
        The flattened DataFrame: the columns of ``data`` other than the column named ``data``, followed by the flattened
        keys (nested keys are joined with a dot, such as ``b.c``).

    Notes
    -----
    The cells are parsed with ``ast.literal_eval``, which means Python literal syntax: JSON ``true``, ``false`` and
    ``null`` are not accepted. The column dropped from the output is the one literally named ``'data'``, whatever
    ``jsonColname`` is, so with another ``jsonColname`` the JSON column itself stays in the result. The flattened keys are
    joined side by side (``pd.concat(axis=1)``) with a new 0 to n-1 index, so ``data`` should have the default index;
    with any other index the rows no longer line up (reset the index first).

    Examples
    --------
    >>> df = pd.DataFrame({'id': [1], 'data': ['{"key1":"val1"}']})
    >>> flatten_json_attr(df)
    """
    """ Flatten Json-format model attributes. """
    import ast
    json_data = data[jsonColname].tolist()
    json_data = [ast.literal_eval(x) for x in json_data]
    info_list = [x for x in data.columns if x != 'data']
    drv = data[info_list]
    drv_w_attr = pd.concat([drv, pd.json_normalize(data=json_data)], axis = 1)
    logging.info(f"Flattened Data Shape: {drv_w_attr.shape}")
    return drv_w_attr

parse_odps_schema

parse_odps_schema(schema_list)

Parse an ODPS schema.

Parameters:

Name	Type	Description	Default
schema_list	list	List of ODPS schema entries, one per column. The string form of each entry must look like <column name, type string>, as the odps.models.Column objects of a table schema (schema.columns) do.	required

Returns:

Type	Description
dict	Dictionary mapping field names to data types.

Examples:

>>> parse_odps_schema(['<column col1, type string>', '<column col2, type bigint>'])
{'col1': 'string', 'col2': 'bigint'}

Source code in Modeling_Tool/Core/utils.py

def parse_odps_schema(schema_list):   
    """
    Parse an ODPS schema.

    Parameters
    ----------
    schema_list : list
        List of ODPS schema entries, one per column. The string form of each entry must look like
        ``<column name, type string>``, as the ``odps.models.Column`` objects of a table schema (``schema.columns``) do.

    Returns
    -------
    dict
        Dictionary mapping field names to data types.

    Examples
    --------
    >>> parse_odps_schema(['<column col1, type string>', '<column col2, type bigint>'])
    {'col1': 'string', 'col2': 'bigint'}
    """
    import re

    fnl_dict = {}
    for x in schema_list:
        res = re.sub(r'[<>]', '', str(x).replace(" type ", "").replace("column ", "")).replace(" ", "").split(',')
        fnl_dict[res[0]] = res[1]
    return fnl_dict

npnan2none

npnan2none(df)

Convert np.nan and np.nat values in a DataFrame to None.

Parameters:

Name	Type	Description	Default
df	DataFrame	Input DataFrame.	required

Returns:

Type	Description
DataFrame	Converted data. Columns that contain missing values end up with object dtype and hold None instead of NaN / NaT.

Notes

The object-dtype columns of the input df are reassigned in place before the conversion, so the caller's frame is modified as well: an object column whose values can all be cast to float (for example numeric strings) is converted to float.

Examples:

>>> df = pd.DataFrame({'a': [1, np.nan], 'b': [np.nan, 2]})
>>> npnan2none(df)

Source code in Modeling_Tool/Core/utils.py

def npnan2none(df):
    """
    Convert np.nan and np.nat values in a DataFrame to None.

    Parameters
    ----------
    df : pandas.DataFrame
        Input DataFrame.

    Returns
    -------
    pandas.DataFrame
        Converted data. Columns that contain missing values end up with ``object`` dtype and hold ``None`` instead of
        NaN / NaT.

    Notes
    -----
    The object-dtype columns of the input ``df`` are reassigned in place before the conversion, so the caller's frame is
    modified as well: an object column whose values can all be cast to float (for example numeric strings) is converted to
    float.

    Examples
    --------
    >>> df = pd.DataFrame({'a': [1, np.nan], 'b': [np.nan, 2]})
    >>> npnan2none(df)
    """
    """ Convert np.nan, np.nat to None value. """

    obj_colist = [k for k, v in df.dtypes.items() if v == 'O']
    for x in obj_colist:
        try:
            df[x] = df[x].astype(float).where(df[x].notnull(), None)
        except:
            df[x] = df[x].astype(object).where(df[x].notnull(), None)

    df = df.replace({np.nan: None})
    return df

drop_tmp_cols

drop_tmp_cols(df, drop_list=['py_inserttime'])

Drop temporary columns from a DataFrame.

Parameters:

Name	Type	Description	Default
df	DataFrame	Input DataFrame.	required
drop_list	list	List of temporary columns to drop. Names that are not columns of df are ignored.	['py_inserttime']

Returns:

Type	Description
DataFrame	Data with the temporary columns dropped (a new frame; df itself is not modified).

Examples:

>>> df = pd.DataFrame({'a': [1, 2], 'py_inserttime': [0, 0]})
>>> drop_tmp_cols(df)

Source code in Modeling_Tool/Core/utils.py

def drop_tmp_cols(df, drop_list = ['py_inserttime']):
    """
    Drop temporary columns from a DataFrame.

    Parameters
    ----------
    df : pandas.DataFrame
        Input DataFrame.
    drop_list : list, default ['py_inserttime']
        List of temporary columns to drop. Names that are not columns of ``df`` are ignored.

    Returns
    -------
    pandas.DataFrame
        Data with the temporary columns dropped (a new frame; ``df`` itself is not modified).

    Examples
    --------
    >>> df = pd.DataFrame({'a': [1, 2], 'py_inserttime': [0, 0]})
    >>> drop_tmp_cols(df)
    """
    """ Drop Temporary Columns. """

    col_exist_dict = {}
    for x in drop_list:
        if x in df.columns:
            col_exist_dict[x] = 1
        else:
            col_exist_dict[x] = 0

    df = df.drop(columns = [k for k, v in col_exist_dict.items() if v == 1])

    return df

mkdir_if_not_exist

mkdir_if_not_exist(folder_path, replace=False)

Create a directory if it does not exist.

Parameters:

Name	Type	Description	Default
folder_path	str	Folder path. Missing parent folders are created as well. None does nothing.	required
replace	bool	Whether to replace the directory if it already exists. Despite the name, an existing directory is neither deleted nor emptied: with replace=True the function only logs that the folder has been replaced and returns 0.	False

Returns:

Type	Description
int or None	Status code: 0 means success, 1 means the directory already exists (and replace=False). None is returned when folder_path is None.

Examples:

>>> mkdir_if_not_exist('/path/to/new/folder')

Source code in Modeling_Tool/Core/utils.py

def mkdir_if_not_exist(folder_path, replace = False):
    """
    Create a directory if it does not exist.

    Parameters
    ----------
    folder_path : str
        Folder path. Missing parent folders are created as well. ``None`` does nothing.
    replace : bool, default False
        Whether to replace the directory if it already exists. Despite the name, an existing directory is neither deleted
        nor emptied: with ``replace=True`` the function only logs that the folder has been replaced and returns 0.

    Returns
    -------
    int or None
        Status code: 0 means success, 1 means the directory already exists (and ``replace=False``). ``None`` is returned
        when ``folder_path`` is ``None``.

    Examples
    --------
    >>> mkdir_if_not_exist('/path/to/new/folder')
    """
    """ Make new directory if the given path does not exist. """

    if folder_path is None:
        return None

    if os.path.isdir(folder_path):

        if replace:
            logging.info(f"Folder {folder_path} has been replaced!")
            os.makedirs(folder_path, exist_ok=replace)
            return 0

#         logging.info(f"Folder {folder_path} has already existed!")
        return 1

    else:
        os.makedirs(folder_path, exist_ok=False)
        logging.info(f"Folder {folder_path} created!")

    return 0

parse_sql_file

parse_sql_file(sql_path: str = None, sql_query: str = None, split: bool = False, format_select: bool = False, **kwargs)

Parse a SQL file and substitute variables.

Parameters:

Name	Type	Description	Default
sql_path	str	Path to the SQL file. Exactly one of sql_path and sql_query must be given.	None
sql_query	str	SQL query string. Exactly one of sql_path and sql_query must be given.	None
split	bool	Whether to split multiple queries.	False
format_select	bool	Whether to automatically format the SELECT fields (one field per line, leading-comma style).	False
**kwargs		Variables in the SQL to substitute: every {name} placeholder is replaced by the value of the keyword argument name, which must be a string.	{}

Returns:

Type	Description
str or list	Parsed SQL string or list of strings. With split=False it is one string: the queries joined with '; ' and ended by ';'. With split=True it is the list of the queries (without semicolons), or the single query string when there is only one.

Raises:

Type	Description
AttributeError	If neither or both of sql_path and sql_query are given.

Notes

Comments are removed first (optimizer hints /*+ ... */ are kept). A placeholder without a matching keyword is left as it is and a UserWarning ("Missing argument(s) ...") is issued.

Examples:

>>> parse_sql_file(sql_path='query.sql', table_name='my_table', date='2025-01-01')
>>> parse_sql_file(sql_path='query.sql', format_select=True, table_name='my_table')

Source code in Modeling_Tool/Core/utils.py

def parse_sql_file(sql_path:str=None,
                   sql_query:str=None,
                   split:bool=False,
                   format_select:bool=False,
                   **kwargs):
    """
    Parse a SQL file and substitute variables.

    Parameters
    ----------
    sql_path : str, optional
        Path to the SQL file. Exactly one of ``sql_path`` and ``sql_query`` must be given.
    sql_query : str, optional
        SQL query string. Exactly one of ``sql_path`` and ``sql_query`` must be given.
    split : bool, default False
        Whether to split multiple queries.
    format_select : bool, default False
        Whether to automatically format the SELECT fields (one field per line, leading-comma style).
    **kwargs
        Variables in the SQL to substitute: every ``{name}`` placeholder is replaced by the value of the keyword argument
        ``name``, which must be a string.

    Returns
    -------
    str or list
        Parsed SQL string or list of strings. With ``split=False`` it is one string: the queries joined with ``'; '`` and
        ended by ``';'``. With ``split=True`` it is the list of the queries (without semicolons), or the single query
        string when there is only one.

    Raises
    ------
    AttributeError
        If neither or both of ``sql_path`` and ``sql_query`` are given.

    Notes
    -----
    Comments are removed first (optimizer hints ``/*+ ... */`` are kept). A placeholder without a matching keyword is left
    as it is and a ``UserWarning`` ("Missing argument(s) ...") is issued.

    Examples
    --------
    >>> parse_sql_file(sql_path='query.sql', table_name='my_table', date='2025-01-01')
    >>> parse_sql_file(sql_path='query.sql', format_select=True, table_name='my_table')
    """
    import re
    import warnings

    if (sql_path is None) and (sql_query is None):
        raise AttributeError("please give either sql_path or sql_query.")

    if (sql_path is not None) and (sql_query is not None):
        raise AttributeError("sql_path and sql_query can not be BOTH given.")

    sql = sql_query
    if sql_path is not None:
        # Read sql file.
        with open(sql_path, 'r') as file:
            sql = file.read().strip()

    # Remove all comments in sql file.
    sql = _remove_comments(sql)

    # Find and parse all argments in sql file.
    all_args = list(set(re.findall(r"{(.*?)}", sql)))
    #### Parse Arguments
    for k, v in kwargs.items():
        if k in all_args:
            sql = sql.replace("{%s}" % k, v)
    args_left = list(set(re.findall(r"{(.*?)}", sql)))

    # Identify if mutliple queries in one .sql file
    ## if Yes: identify if split sql file.
    query_list = _split_sql_queries(sql)
    query_list = [query.strip() for query in query_list if query != '']

    # Optionally format SELECT clause for readability
    if format_select:
        query_list = [_format_sql_select(q) for q in query_list]

    if len(args_left) != 0:
        # Raise a warning if not all arguments are given through the function.
        warnings.warn(f"Missing argument(s) {', '.join(args_left)} in the given SQL file.")

    # Ensure all the arguments have been claimed.
    if split:
        return query_list if len(query_list) > 1 else query_list[0]
    else:
        return '; '.join(query_list)+";"

calc_woe

calc_woe(data, bad_pct, good_pct, fillwoe=True)

Compute the weight of evidence (WOE).

WOE = ln(the bin's share of all bad samples / the bin's share of all good samples)

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the proportions.	required
bad_pct	str	Name of the column holding the bad-sample proportion.	required
good_pct	str	Name of the column holding the good-sample proportion.	required
fillwoe	bool	Value returned when data has no rows: 0 if True, NaN if False. It does not change the result for rows whose proportion is 0 (see Notes).	True

Returns:

Type	Description
float or Series	WOE value(s): a Series aligned with data (numpy.log(bad_pct / good_pct) row by row), or the scalar 0 / NaN (see fillwoe) when data is empty.

Notes

A row in which a proportion is 0 is not filled: a zero bad proportion gives -inf, a zero good proportion gives inf and two zeros give NaN, whatever fillwoe is.

Examples:

>>> df = pd.DataFrame({'bad_pct': [0.3, 0.5], 'good_pct': [0.7, 0.5]})
>>> calc_woe(df, 'bad_pct', 'good_pct')

Source code in Modeling_Tool/Core/utils.py

def calc_woe(data, bad_pct, good_pct, fillwoe=True):
    """
    Compute the weight of evidence (WOE).

    WOE = ln(the bin's share of all bad samples / the bin's share of all good samples)

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the proportions.
    bad_pct : str
        Name of the column holding the bad-sample proportion.
    good_pct : str
        Name of the column holding the good-sample proportion.
    fillwoe : bool, default True
        Value returned when ``data`` has no rows: 0 if True, NaN if False. It does not change the result for rows whose
        proportion is 0 (see Notes).

    Returns
    -------
    float or pandas.Series
        WOE value(s): a Series aligned with ``data`` (``numpy.log(bad_pct / good_pct)`` row by row), or the scalar 0 / NaN
        (see ``fillwoe``) when ``data`` is empty.

    Notes
    -----
    A row in which a proportion is 0 is not filled: a zero bad proportion gives ``-inf``, a zero good proportion gives
    ``inf`` and two zeros give NaN, whatever ``fillwoe`` is.

    Examples
    --------
    >>> df = pd.DataFrame({'bad_pct': [0.3, 0.5], 'good_pct': [0.7, 0.5]})
    >>> calc_woe(df, 'bad_pct', 'good_pct')
    """

    return _calc_woe_iv_values(data, bad_pct, good_pct, fillwoe=fillwoe)[0]

calc_iv

calc_iv(data, bad_pct, good_pct, filliv=True)

Compute the information value (IV).

IV = (the bin's share of all bad samples - the bin's share of all good samples) * WOE

Parameters:

Name	Type	Description	Default
data	DataFrame	DataFrame containing the proportions.	required
bad_pct	str	Name of the column holding the bad-sample proportion.	required
good_pct	str	Name of the column holding the good-sample proportion.	required
filliv	bool	Value returned when data has no rows: 0 if True, NaN if False. It does not change the result for rows whose proportion is 0 (see Notes).	True

Returns:

Type	Description
float or Series	IV value(s): a Series aligned with data ((bad_pct - good_pct) * WOE row by row), or the scalar 0 / NaN (see filliv) when data is empty.

Notes

A row in which one proportion is 0 is not filled: its IV is inf (NaN when both proportions are 0), whatever filliv is.

Examples:

>>> df = pd.DataFrame({'bad_pct': [0.3, 0.5], 'good_pct': [0.7, 0.5]})
>>> calc_iv(df, 'bad_pct', 'good_pct')

Source code in Modeling_Tool/Core/utils.py

def calc_iv(data, bad_pct, good_pct, filliv=True):
    """
    Compute the information value (IV).

    IV = (the bin's share of all bad samples - the bin's share of all good samples) * WOE

    Parameters
    ----------
    data : pandas.DataFrame
        DataFrame containing the proportions.
    bad_pct : str
        Name of the column holding the bad-sample proportion.
    good_pct : str
        Name of the column holding the good-sample proportion.
    filliv : bool, default True
        Value returned when ``data`` has no rows: 0 if True, NaN if False. It does not change the result for rows whose
        proportion is 0 (see Notes).

    Returns
    -------
    float or pandas.Series
        IV value(s): a Series aligned with ``data`` (``(bad_pct - good_pct) * WOE`` row by row), or the scalar 0 / NaN (see
        ``filliv``) when ``data`` is empty.

    Notes
    -----
    A row in which one proportion is 0 is not filled: its IV is ``inf`` (NaN when both proportions are 0), whatever
    ``filliv`` is.

    Examples
    --------
    >>> df = pd.DataFrame({'bad_pct': [0.3, 0.5], 'good_pct': [0.7, 0.5]})
    >>> calc_iv(df, 'bad_pct', 'good_pct')
    """

    return _calc_woe_iv_values(data, bad_pct, good_pct, filliv=filliv)[1]

save_model

save_model(model, filename)

Save a LightGBM model using pickle.

Parameters:

Name	Type	Description	Default
model	object	Model object to save.	required
filename	str	Path to save the model to.	required

Returns:

Type	Description
int	Execution status code (0 means success).

Notes

The object is written with joblib.dump exactly as given, with no SMF artifact envelope and no metadata, so any picklable object works, not only LightGBM models. The exported Modeling_Tool.save_model is the metadata-aware version from Modeling_Tool.Core.Model_Registry_Tool.

Examples:

>>> save_model(model, 'model.pkl')

Source code in Modeling_Tool/Core/utils.py

def save_model(model, filename):
    """
    Save a LightGBM model using pickle.

    Parameters
    ----------
    model : object
        Model object to save.
    filename : str
        Path to save the model to.

    Returns
    -------
    int
        Execution status code (0 means success).

    Notes
    -----
    The object is written with ``joblib.dump`` exactly as given, with no SMF artifact envelope and no metadata, so any
    picklable object works, not only LightGBM models. The exported ``Modeling_Tool.save_model`` is the metadata-aware
    version from ``Modeling_Tool.Core.Model_Registry_Tool``.

    Examples
    --------
    >>> save_model(model, 'model.pkl')
    """
    """ Save lightGBM model using pickle. """
    import joblib

    joblib.dump(model, filename)
    return 0

load_model

load_model(model_path)

Load a pickled model.

Parameters:

Name	Type	Description	Default
model_path	str	Path to the model file.	required

Returns:

Type	Description
object	The loaded model object.

Notes

The file is read with joblib.load and the object is returned exactly as stored: a file written by the metadata-aware Modeling_Tool.save_model comes back as its artifact dict, not as the model (use Modeling_Tool.load_model for that). Only load files from a trusted source.

Examples:

>>> model = load_model('model.pkl')

Source code in Modeling_Tool/Core/utils.py

def load_model(model_path):
    """
    Load a pickled model.

    Parameters
    ----------
    model_path : str
        Path to the model file.

    Returns
    -------
    object
        The loaded model object.

    Notes
    -----
    The file is read with ``joblib.load`` and the object is returned exactly as stored: a file written by the
    metadata-aware ``Modeling_Tool.save_model`` comes back as its artifact dict, not as the model (use
    ``Modeling_Tool.load_model`` for that). Only load files from a trusted source.

    Examples
    --------
    >>> model = load_model('model.pkl')
    """
    """ Load Pickle Model. """
    import joblib

    model = joblib.load(model_path)
    return model

scoring

scoring(data, model, varlist, scr_name, keeplist=None, all_missing_spec_value=None)

Score data with a model.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
model	sklearn-like model	Machine learning model. It must provide predict_proba; the score is the second column of its output (the probability of class 1).	required
varlist	list	List of feature variables.	required
scr_name	str	Name of the score column (an existing column with this name is overwritten).	required
keeplist	list	List of columns to keep. The score column scr_name is always appended to it. If None, all the columns of data and the score column are returned.	None
all_missing_spec_value	float	Score value to assign to samples in which all features are missing. It is applied only when it is truthy, so 0 is treated as not given.	None

Returns:

Type	Description
DataFrame	DataFrame containing the scores.

Notes

data itself is not modified. When some rows have all varlist features missing, those rows are moved behind the other rows (their index labels are kept), so the row order of the result can differ from data.

Examples:

>>> df = scoring(data, model, ['feat1', 'feat2'], 'score')

Source code in Modeling_Tool/Core/utils.py

def scoring(data, model, varlist, scr_name, keeplist = None, all_missing_spec_value = None):
    """
    Score data with a model.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    model : sklearn-like model
        Machine learning model. It must provide ``predict_proba``; the score is the second column of its output (the
        probability of class 1).
    varlist : list
        List of feature variables.
    scr_name : str
        Name of the score column (an existing column with this name is overwritten).
    keeplist : list, optional
        List of columns to keep. The score column ``scr_name`` is always appended to it. If None, all the columns of
        ``data`` and the score column are returned.
    all_missing_spec_value : float, optional
        Score value to assign to samples in which all features are missing. It is applied only when it is truthy, so ``0``
        is treated as not given.

    Returns
    -------
    pandas.DataFrame
        DataFrame containing the scores.

    Notes
    -----
    ``data`` itself is not modified. When some rows have all ``varlist`` features missing, those rows are moved behind the
    other rows (their index labels are kept), so the row order of the result can differ from ``data``.

    Examples
    --------
    >>> df = scoring(data, model, ['feat1', 'feat2'], 'score')
    """
    """ Model Soring. """

    fnl_data = data.copy()
    fnl_data[scr_name] = model.predict_proba(fnl_data.loc[:, varlist])[:, 1]

    nohit_condition = (pd.isnull(fnl_data[varlist]).sum(axis = 1) == len(varlist))
    if fnl_data[nohit_condition].shape[0] > 0:

        all_missing_data = fnl_data[nohit_condition].copy()
        other_data = fnl_data[~nohit_condition]


        all_missing_data[scr_name] = model.predict_proba(fnl_data[nohit_condition].loc[:, varlist])[:, 1]
        logger.info("Score for All-Missing Cases: %s", all_missing_data[scr_name].unique())

        if all_missing_spec_value:
            all_missing_data[scr_name] = all_missing_spec_value
            logger.info(f"Score for All-Missing Cases Has Been Reset to {all_missing_spec_value}")

        fnl_data = pd.concat([other_data, all_missing_data])

    assert fnl_data.shape[0] == data.shape[0]

    if keeplist is None:
        keeplist = fnl_data.columns.tolist()
    else:
        keeplist = keeplist + [scr_name]

    return fnl_data[keeplist]

get_missing_indicator

get_missing_indicator(data, subset=None)

Add Missing Indicator.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
subset	list of str	Names of the columns to check. Required in practice: None raises KeyError.	None

Returns:

Type	Description
Series	Integer indicator aligned with data: 1 when all the columns of subset are missing in the row, otherwise 0.

Notes

The indicator is only returned: no column is added to data.

Source code in Modeling_Tool/Core/utils.py

def get_missing_indicator(data, subset = None):
    """
    Add Missing Indicator.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    subset : list of str, default None
        Names of the columns to check. Required in practice: ``None`` raises ``KeyError``.

    Returns
    -------
    pandas.Series
        Integer indicator aligned with ``data``: 1 when all the columns of ``subset`` are missing in the row, otherwise 0.

    Notes
    -----
    The indicator is only returned: no column is added to ``data``.
    """

    all_missing_logic = lambda data: (pd.isnull(data[subset]).sum(axis = 1) == len(subset))
    return all_missing_logic(data).astype(int)

upload_score

upload_score(data, model, varlist, scr_name, table_name, keeplist=None, retPandas=False, all_missing_spec_value=None)

Upload model scores to MaxCompute.

Parameters:

Name	Type	Description	Default
data	DataFrame	Input DataFrame.	required
model	sklearn-like model	Machine learning model.	required
varlist	list	List of feature variables.	required
scr_name	str	Name of the score column.	required
table_name	str	Name of the target table.	required
keeplist	list	List of columns to keep. The score column scr_name is always appended to it.	None
retPandas	bool	Whether to return the pandas DataFrame.	False
all_missing_spec_value	float	Score value to assign to samples in which all features are missing (see scoring).	None

Returns:

Type	Description
int or DataFrame	Status code (0) or, when retPandas=True, the DataFrame that was uploaded.

Notes

The data are scored with scoring, the missing values are converted to None (npnan2none), the temporary column py_inserttime is dropped (drop_tmp_cols) and the result is uploaded with ODPSRunner.upload_df, which replaces the table table_name. Creating the ODPSRunner needs the odps extra and the ALIBABA_CLOUD_ACCESS_KEY_ID / ALIBABA_CLOUD_ACCESS_KEY_SECRET environment variables.

Examples:

>>> upload_score(data, model, ['feat1', 'feat2'], 'score', 'output_table')

Source code in Modeling_Tool/Core/utils.py

def upload_score(data, model, varlist, scr_name, table_name, keeplist = None, retPandas = False, all_missing_spec_value = None):
    """
    Upload model scores to MaxCompute.

    Parameters
    ----------
    data : pandas.DataFrame
        Input DataFrame.
    model : sklearn-like model
        Machine learning model.
    varlist : list
        List of feature variables.
    scr_name : str
        Name of the score column.
    table_name : str
        Name of the target table.
    keeplist : list, optional
        List of columns to keep. The score column ``scr_name`` is always appended to it.
    retPandas : bool, default False
        Whether to return the pandas DataFrame.
    all_missing_spec_value : float, optional
        Score value to assign to samples in which all features are missing (see ``scoring``).

    Returns
    -------
    int or pandas.DataFrame
        Status code (0) or, when ``retPandas=True``, the DataFrame that was uploaded.

    Notes
    -----
    The data are scored with ``scoring``, the missing values are converted to ``None`` (``npnan2none``), the temporary
    column ``py_inserttime`` is dropped (``drop_tmp_cols``) and the result is uploaded with ``ODPSRunner.upload_df``,
    which replaces the table ``table_name``. Creating the ``ODPSRunner`` needs the ``odps`` extra and the
    ``ALIBABA_CLOUD_ACCESS_KEY_ID`` / ``ALIBABA_CLOUD_ACCESS_KEY_SECRET`` environment variables.

    Examples
    --------
    >>> upload_score(data, model, ['feat1', 'feat2'], 'score', 'output_table')
    """
    """ Upload Score to Maxcompute. """

    data = scoring(data = data, model = model, varlist = varlist, scr_name = scr_name, keeplist = keeplist, all_missing_spec_value = all_missing_spec_value)


    from .ODPS_Tool import ODPSRunner

    sqlrunner = ODPSRunner()

    fnl_scr_upload = data.copy()
    fnl_scr_upload = npnan2none(fnl_scr_upload)
    fnl_scr_upload = drop_tmp_cols(fnl_scr_upload)

    if keeplist is None:
        keeplist = fnl_scr_upload.columns.tolist()
    else:
        keeplist = keeplist + [scr_name]

    sqlrunner.upload_df(fnl_scr_upload[keeplist], table_name)

    if retPandas:
        return fnl_scr_upload[keeplist]

    return 0

pull_attributes_in_batch

pull_attributes_in_batch(table_name, varlist, batch_num=6, unikey='flow_id', main_info_select=['*'], add_query='')

Pull Data from DataWorks in Vertical Batch.

The variables are cut into column batches (cut2pieces) and every batch is downloaded with its own query, which also selects unikey. The remaining columns come from a separate query and all pieces are merged on unikey.

Parameters:

Name	Type	Description	Default
table_name	str	Name of the source table.	required
varlist	list of str	Names of the attribute columns to pull in vertical batches. At least 6 names are needed (see Notes).	required
batch_num	int	Not used: the number of batches is fixed to 6 inside the function, whatever value is passed.	6
unikey	str	Name of the unique key column. It is selected in every batch and used to merge the pieces.	'flow_id'
main_info_select	list of str	Select items of the query that fetches the columns that are not in varlist. The query is built as SELECT <items joined by ','> EXCEPT (<varlist>) FROM <table_name> <add_query>, so it is meant to hold '*' (the default), and the result must contain unikey.	['*']
add_query	str	SQL text appended after the table name in every query (for example a WHERE clause).	''

Returns:

Type	Description
DataFrame	The result of the main-info query merged (inner join on unikey) with the columns of every batch.

Notes

It needs an ODPSRunner (the odps extra and the ALIBABA_CLOUD_ACCESS_KEY_ID / ALIBABA_CLOUD_ACCESS_KEY_SECRET environment variables) and downloads with cpu_count() - 1 processes. varlist is cut with cut2pieces(varlist, 6), which raises ValueError for fewer than 6 names and does not always return exactly 6 batches.

Source code in Modeling_Tool/Core/utils.py

def pull_attributes_in_batch(table_name, varlist, batch_num = 6, unikey = 'flow_id', main_info_select = ['*'], add_query = ''):
    """
    Pull Data from DataWorks in Vertical Batch.

    The variables are cut into column batches (``cut2pieces``) and every batch is downloaded with its own query, which
    also selects ``unikey``. The remaining columns come from a separate query and all pieces are merged on ``unikey``.

    Parameters
    ----------
    table_name : str
        Name of the source table.
    varlist : list of str
        Names of the attribute columns to pull in vertical batches. At least 6 names are needed (see Notes).
    batch_num : int, default 6
        Not used: the number of batches is fixed to 6 inside the function, whatever value is passed.
    unikey : str, default 'flow_id'
        Name of the unique key column. It is selected in every batch and used to merge the pieces.
    main_info_select : list of str, default ['*']
        Select items of the query that fetches the columns that are not in ``varlist``. The query is built as
        ``SELECT <items joined by ','> EXCEPT (<varlist>) FROM <table_name> <add_query>``, so it is meant to hold
        ``'*'`` (the default), and the result must contain ``unikey``.
    add_query : str, default ''
        SQL text appended after the table name in every query (for example a ``WHERE`` clause).

    Returns
    -------
    pandas.DataFrame
        The result of the main-info query merged (inner join on ``unikey``) with the columns of every batch.

    Notes
    -----
    It needs an ``ODPSRunner`` (the ``odps`` extra and the ``ALIBABA_CLOUD_ACCESS_KEY_ID`` /
    ``ALIBABA_CLOUD_ACCESS_KEY_SECRET`` environment variables) and downloads with ``cpu_count() - 1`` processes.
    ``varlist`` is cut with ``cut2pieces(varlist, 6)``, which raises ``ValueError`` for fewer than 6 names and does not
    always return exactly 6 batches.
    """

    from .ODPS_Tool import ODPSRunner
    import multiprocessing

    n_process = multiprocessing.cpu_count() - 1
    logger.info(n_process)
    sqlrunner = ODPSRunner()

    n = 6

    batch_varlist = cut2pieces(varlist, n)    

    assert (len(varlist) == len([x for varlist in batch_varlist for x in varlist]))

    res = {}
    i = 0
    while i < len(batch_varlist):

        logger.info(i)
        sql_query = f""" 
            SELECT {unikey}, {", ".join(batch_varlist[i])} 
            FROM {table_name}
            {add_query};
        """

        res[f'batch{i}'] = sqlrunner.run_sql(sql_query, n_process = n_process)

        i += 1

    sql_query = f""" 
        SELECT {','.join(main_info_select)} EXCEPT ({", ".join(varlist)}) 
        FROM {table_name}
        {add_query};
    """

    main_info = sqlrunner.run_sql(sql_query, n_process = n_process)
    master_df = main_info.copy()

    for k, data in res.items():    
        master_df = master_df.merge(data, on = [unikey])

    return master_df

get_feature_names

get_feature_names(model, model_type=None)

Get the list of feature names of a model.

Detect the model type automatically and return its feature names. Supports LightGBM, XGBoost, sklearn and other models.

Parameters:

Name	Type	Description	Default
model	object	Trained machine learning model object.	required
model_type	str	Model type hint, compared case-insensitively. Allowed values: 'lgb' or 'lightgbm': LightGBM model 'xgb' or 'xgboost': XGBoost model 'sklearn': sklearn model (the type is detected automatically, as with None) None: detect automatically (default) Any other string is handled like None.	None

Returns:

Type	Description
list	List of feature names.

Raises:

Type	Description
ValueError	Raised when the feature names cannot be obtained.

Examples:

>>> # Generic usage
>>> feature_names = get_feature_names(model)

>>> # Specify the model type
>>> feature_names = get_feature_names(lgb_model, model_type='lgb')

>>> # XGBoost model
>>> feature_names = get_feature_names(xgb_model, model_type='xgb')

Source code in Modeling_Tool/Core/utils.py

def get_feature_names(model, model_type=None):
    """
    Get the list of feature names of a model.

    Detect the model type automatically and return its feature names.
    Supports LightGBM, XGBoost, sklearn and other models.

    Parameters
    ----------
    model : object
        Trained machine learning model object.
    model_type : str, optional
        Model type hint, compared case-insensitively. Allowed values:

        - 'lgb' or 'lightgbm': LightGBM model
        - 'xgb' or 'xgboost': XGBoost model
        - 'sklearn': sklearn model (the type is detected automatically, as with None)
        - None: detect automatically (default)

        Any other string is handled like None.

    Returns
    -------
    list
        List of feature names.

    Raises
    ------
    ValueError
        Raised when the feature names cannot be obtained.

    Examples
    --------
    >>> # Generic usage
    >>> feature_names = get_feature_names(model)

    >>> # Specify the model type
    >>> feature_names = get_feature_names(lgb_model, model_type='lgb')

    >>> # XGBoost model
    >>> feature_names = get_feature_names(xgb_model, model_type='xgb')
    """
    # If a model type is specified, prefer the dedicated function
    if model_type is not None:
        model_type_lower = model_type.lower()
        if model_type_lower in ['lgb', 'lightgbm']:
            return get_feature_names_lgb(model)
        elif model_type_lower in ['xgb', 'xgboost']:
            return get_feature_names_xgb(model)

    # Detect the model type automatically and get the feature names
    model_class_name = model.__class__.__name__.lower()

    # SMF GradientBoostingModel wraps the fitted estimator in _model.model and
    # stores DataFrame column names on _model.feature_names_ after fit.
    wrapped_model_type = getattr(model, 'model_type', None)
    wrapped_backend = getattr(model, '_model', None)
    if wrapped_model_type is not None and wrapped_backend is not None:
        wrapped_feature_names = getattr(wrapped_backend, 'feature_names_', None)
        if wrapped_feature_names is not None:
            return list(wrapped_feature_names)

        wrapped_estimator = getattr(wrapped_backend, 'model', None)
        if wrapped_estimator is not None:
            wrapped_model_type_lower = str(wrapped_model_type).lower()
            if wrapped_model_type_lower in ['lgb', 'lightgbm']:
                return get_feature_names_lgb(wrapped_estimator)
            if wrapped_model_type_lower in ['xgb', 'xgboost']:
                return get_feature_names_xgb(wrapped_estimator)
            return get_feature_names(wrapped_estimator)

    # LightGBM detection
    if 'lgb' in model_class_name or 'lightgbm' in model_class_name:
        return get_feature_names_lgb(model)

    # XGBoost detection
    if 'xgb' in model_class_name or 'xgboost' in model_class_name:
        return get_feature_names_xgb(model)

    if 'logisticregression' in model_class_name:
        return list(model.feature_names_in_)

    # Try the generic sklearn approaches
    # Method 1: feature_names_in_ attribute (sklearn >= 1.0)
    if hasattr(model, 'feature_names_in_'):
        return list(model.feature_names_in_)

    # Method 2: feature_names attribute
    if hasattr(model, 'feature_names'):
        feature_names = model.feature_names
        if callable(feature_names):
            return list(feature_names())
        return list(feature_names)

    # Method 3: booster approach (LightGBM-specific)
    if hasattr(model, 'booster_'):
        try:
            return model.booster_.feature_name()
        except (AttributeError, TypeError):
            pass

    # Method 4: try to get the names from the model parameters
    if hasattr(model, 'feature_name'):
        try:
            feature_names = model.feature_name
            if callable(feature_names):
                return list(feature_names())
            return list(feature_names)
        except (AttributeError, TypeError):
            pass

    # Unable to get the feature names
    raise ValueError(
        f"Cannot get the feature names of model '{model_class_name}'.\n"
        f"Please try:\n"
        f"1. Specify the model_type argument explicitly\n"
        f"2. Use a dedicated function: get_feature_names_lgb() or get_feature_names_xgb()"
    )

get_feature_names_lgb

get_feature_names_lgb(model)

Get the feature names of a LightGBM model.

Parameters:

Name	Type	Description	Default
model	LGBMClassifier or LGBMRegressor	Trained LightGBM model.	required

Returns:

Type	Description
list	List of feature names.

Raises:

Type	Description
ValueError	Raised when the feature names cannot be obtained.

Examples:

>>> import lightgbm as lgb
>>> model = lgb.LGBMClassifier().fit(X_train, y_train)
>>> feature_names = get_feature_names_lgb(model)
>>> print(feature_names)
['feature_1', 'feature_2', 'feature_3']

Source code in Modeling_Tool/Core/utils.py

def get_feature_names_lgb(model):
    """Get the feature names of a LightGBM model.

    Parameters
    ----------
    model : lgb.LGBMClassifier or lgb.LGBMRegressor
        Trained LightGBM model.

    Returns
    -------
    list
        List of feature names.

    Raises
    ------
    ValueError
        Raised when the feature names cannot be obtained.

    Examples
    --------
    >>> import lightgbm as lgb
    >>> model = lgb.LGBMClassifier().fit(X_train, y_train)
    >>> feature_names = get_feature_names_lgb(model)
    >>> print(feature_names)
    ['feature_1', 'feature_2', 'feature_3']
    """
    # Method 1: booster_.feature_name() (most reliable)
    if hasattr(model, 'booster_') and model.booster_ is not None:
        try:
            return model.booster_.feature_name()
        except (AttributeError, TypeError):
            pass

    # Method 2: feature_name_ attribute
    if hasattr(model, 'feature_name_'):
        return list(model.feature_name_)

    # Method 3: feature_name attribute/method
    if hasattr(model, 'feature_name'):
        feature_names = model.feature_name
        if callable(feature_names):
            return list(feature_names())
        return list(feature_names)

    raise ValueError(
        "Cannot get the feature names of the LightGBM model.\n"
        "Make sure the model has been trained correctly."
    )

get_feature_names_xgb

get_feature_names_xgb(model)

Get the feature names of an XGBoost model.

Parameters:

Name	Type	Description	Default
model	XGBClassifier or XGBRegressor	Trained XGBoost model.	required

Returns:

Type	Description
list	List of feature names (an empty list when the booster carries no feature names).

Raises:

Type	Description
ValueError	Raised when the feature names cannot be obtained.

Examples:

>>> import xgboost as xgb
>>> model = xgb.XGBClassifier().fit(X_train, y_train)
>>> feature_names = get_feature_names_xgb(model)
>>> print(feature_names)
['feature_1', 'feature_2', 'feature_3']

Source code in Modeling_Tool/Core/utils.py

def get_feature_names_xgb(model):
    """
    Get the feature names of an XGBoost model.

    Parameters
    ----------
    model : xgb.XGBClassifier or xgb.XGBRegressor
        Trained XGBoost model.

    Returns
    -------
    list
        List of feature names (an empty list when the booster carries no feature names).

    Raises
    ------
    ValueError
        Raised when the feature names cannot be obtained.

    Examples
    --------
    >>> import xgboost as xgb
    >>> model = xgb.XGBClassifier().fit(X_train, y_train)
    >>> feature_names = get_feature_names_xgb(model)
    >>> print(feature_names)
    ['feature_1', 'feature_2', 'feature_3']
    """
    # Method 1: feature_names_in_ attribute (sklearn style)
    if hasattr(model, 'feature_names_in_'):
        return list(model.feature_names_in_)

    # Method 2: booster.get_feature_names() (native XGBoost)
    if hasattr(model, 'get_booster'):
        try:
            booster = model.get_booster()
            feature_names = booster.get_feature_names()
            return list(feature_names) if feature_names else []
        except (AttributeError, TypeError):
            pass

    # Method 3: feature_names attribute
    if hasattr(model, 'feature_names'):
        feature_names = model.feature_names
        if callable(feature_names):
            return list(feature_names())
        return list(feature_names)

    raise ValueError(
        "Cannot get the feature names of the XGBoost model.\n"
        "Make sure the model has been trained correctly."
    )

get_feature_names_batch

get_feature_names_batch(models, model_type=None)

Get the feature names of multiple models in batch.

Parameters:

Name	Type	Description	Default
models	dict or list	Model dictionary {name: model} or list of models.	required
model_type	str	Model type hint (see get_feature_names).	None

Returns:

Type	Description
dict or list	Dictionary or list of feature names, matching the structure of the input.

Raises:

Type	Description
TypeError	If models is neither a dict nor a list.

Examples:

>>> models = {'lgb': lgb_model, 'xgb': xgb_model}
>>> feature_names_dict = get_feature_names_batch(models)
>>> print(feature_names_dict)
{'lgb': ['f1', 'f2'], 'xgb': ['f1', 'f2']}

Source code in Modeling_Tool/Core/utils.py

def get_feature_names_batch(models, model_type=None):
    """
    Get the feature names of multiple models in batch.

    Parameters
    ----------
    models : dict or list
        Model dictionary {name: model} or list of models.
    model_type : str, optional
        Model type hint (see ``get_feature_names``).

    Returns
    -------
    dict or list
        Dictionary or list of feature names, matching the structure of the input.

    Raises
    ------
    TypeError
        If ``models`` is neither a dict nor a list.

    Examples
    --------
    >>> models = {'lgb': lgb_model, 'xgb': xgb_model}
    >>> feature_names_dict = get_feature_names_batch(models)
    >>> print(feature_names_dict)
    {'lgb': ['f1', 'f2'], 'xgb': ['f1', 'f2']}
    """
    if isinstance(models, dict):
        return {
            name: get_feature_names(model, model_type=model_type)
            for name, model in models.items()
        }
    elif isinstance(models, list):
        return [get_feature_names(model, model_type=model_type) for model in models]
    else:
        raise TypeError("The models argument must be a dict or a list")

Encryption: XOR_Encryptor

XOR_Encryptor

TextEncryptor

Text encryption and decryption utility class based on the XOR algorithm.

The class encrypts and decrypts single strings and entire pandas DataFrames. Encrypted data is encoded as URL-safe Base64, which makes it easy to store and transmit. This is obfuscation, not strong cryptography: do not rely on it to protect secrets.

Parameters:

Name	Type	Description	Default
key	str	Key used for encryption and decryption. A key is required in practice: with None (the default) encrypt raises TypeError, with an empty string it raises ZeroDivisionError, and decrypt raises ValueError in both cases.	None
suffix	str	Suffix that encrypt_dataframe appends to the column names. decrypt_dataframe removes every occurrence of it from the column names.	'_encrypted'

Attributes:

Name	Type	Description
key	str	The key given at construction.
suffix	str	The column-name suffix given at construction.

Examples:

>>> encryptor = TextEncryptor(key="my_secret_key")
>>> encrypted = encryptor.encrypt("Hello World")
>>> encryptor.decrypt(encrypted)
'Hello World'

Source code in Modeling_Tool/Core/XOR_Encryptor.py

class TextEncryptor:
    """
    Text encryption and decryption utility class based on the XOR algorithm.

    The class encrypts and decrypts single strings and entire pandas DataFrames. Encrypted data is encoded as URL-safe
    Base64, which makes it easy to store and transmit. This is obfuscation, not strong cryptography: do not rely on it to
    protect secrets.

    Parameters
    ----------
    key : str, default None
        Key used for encryption and decryption. A key is required in practice: with ``None`` (the default) ``encrypt``
        raises ``TypeError``, with an empty string it raises ``ZeroDivisionError``, and ``decrypt`` raises ``ValueError``
        in both cases.
    suffix : str, default '_encrypted'
        Suffix that ``encrypt_dataframe`` appends to the column names. ``decrypt_dataframe`` removes every occurrence of
        it from the column names.

    Attributes
    ----------
    key : str
        The key given at construction.
    suffix : str
        The column-name suffix given at construction.

    Examples
    --------
    >>> encryptor = TextEncryptor(key="my_secret_key")
    >>> encrypted = encryptor.encrypt("Hello World")
    >>> encryptor.decrypt(encrypted)
    'Hello World'
    """

    def __init__(self, key=None, suffix='_encrypted'):
        """Store the key and the column-name suffix (see the class docstring for the parameters)."""
        self.key = key
        self.suffix = suffix

    def encrypt(self, text):
        """
        Encrypt a string.

        The UTF-8 bytes of ``text`` are XORed with the repeated key and prefixed with 2 bytes that hold the length of
        the plaintext in bytes, which ``decrypt`` uses for validation. The result is encoded as URL-safe Base64.

        Parameters
        ----------
        text : str
            Plaintext to encrypt, at most 65,535 bytes of UTF-8.

        Returns
        -------
        str
            Encrypted text, URL-safe Base64.

        Raises
        ------
        TypeError
            If the key is None.
        ZeroDivisionError
            If the key is an empty string.
        OverflowError
            If ``text`` is longer than 65,535 bytes.

        Examples
        --------
        >>> encryptor = TextEncryptor(key="secret")
        >>> encryptor.encrypt("Hello")
        'AAU7AA8eCg=='
        """
        # Text to bytes
        text_bytes = text.encode('utf-8')

        # Expand byte length
        key_bytes = (self.key * (len(text_bytes) // len(self.key) + 1)).encode('utf-8')
        key_bytes = key_bytes[:len(text_bytes)]

        # XOR encryption
        encrypted_bytes = bytes([text_bytes[i] ^ key_bytes[i] for i in range(len(text_bytes))])

        # Add 2 more bytes for verification: the plaintext length in bytes, which is what decrypt compares against
        length_byte = len(text_bytes).to_bytes(2, 'big')

        # combine
        result_bytes = length_byte + encrypted_bytes
        return base64.urlsafe_b64encode(result_bytes).decode('utf-8')

    def decrypt(self, encrypted_text):
        """
        Decrypt a string produced by ``encrypt``.

        The Base64 text is decoded, the first 2 bytes give the stored plaintext length, and the remaining bytes are
        XORed with the repeated key. The length of the recovered bytes is checked against the stored length.

        Parameters
        ----------
        encrypted_text : str
            URL-safe Base64 text produced by ``encrypt``.

        Returns
        -------
        str
            The original plaintext.

        Raises
        ------
        ValueError
            If decryption fails for any reason: the input is not valid Base64, the key is wrong or missing, the stored
            length does not match the recovered length (the data was altered), or the recovered bytes are not valid
            UTF-8.

        Examples
        --------
        >>> encryptor = TextEncryptor(key="secret")
        >>> encryptor.decrypt(encryptor.encrypt("Hello"))
        'Hello'
        """
        try:
            # b64 decryption
            decoded_bytes = base64.urlsafe_b64decode(encrypted_text.encode('utf-8'))

            # extract length info
            text_length = int.from_bytes(decoded_bytes[:2], 'big')

            # extract encryption info
            encrypted_bytes = decoded_bytes[2:]

            # regenerate key
            key_bytes = (self.key * (len(encrypted_bytes) // len(self.key) + 1)).encode('utf-8')
            key_bytes = key_bytes[:len(encrypted_bytes)]

            # XOR Decrypt
            decrypted_bytes = bytes([encrypted_bytes[i] ^ key_bytes[i] for i in range(len(encrypted_bytes))])

            # Check Length
            if len(decrypted_bytes) != text_length:
                raise ValueError("Text Length Does Not Match!")

            return decrypted_bytes.decode('utf-8')
        except:
            raise ValueError("Decrypt Failed! Data Might be Destroyed or Incorrect Key!")

    def encrypt_dataframe(self, data):
        """
        Encrypt every value of a DataFrame.

        All values are converted to strings (a missing value becomes ``'nan'``) and encrypted with ``encrypt``, and the
        suffix is appended to every column name. The original DataFrame is not modified.

        Parameters
        ----------
        data : pandas.DataFrame
            The DataFrame to encrypt.

        Returns
        -------
        pandas.DataFrame
            A new DataFrame of encrypted strings (object dtype) whose column names carry the suffix.

        Raises
        ------
        TypeError
            If the key is None.
        ZeroDivisionError
            If the key is an empty string.
        OverflowError
            If a value, converted to a string, is longer than 65,535 bytes.

        Notes
        -----
        An encrypted DataFrame cannot be analyzed; decrypt it first. A column name that already contains the suffix text
        does not survive a round trip, because ``decrypt_dataframe`` removes every occurrence of the suffix.

        Examples
        --------
        >>> import pandas as pd
        >>> df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})
        >>> encryptor = TextEncryptor(key="secret")
        >>> encryptor.encrypt_dataframe(df).columns.tolist()
        ['name_encrypted', 'age_encrypted']
        """
        res = data.copy()
        collist = data.columns.tolist()
        for col in collist:
            ## Encryption
            res[col] = res[col].astype(str)
            res[col] = res[col].apply(lambda x: self.encrypt(x))
        res.columns = [x + self.suffix for x in res.columns]
        return res

    def decrypt_dataframe(self, data):
        """
        Decrypt a DataFrame produced by ``encrypt_dataframe``.

        Every value is decrypted with ``decrypt`` and every occurrence of the suffix is removed from the column names.
        The original DataFrame is not modified.

        Parameters
        ----------
        data : pandas.DataFrame
            A DataFrame produced by ``encrypt_dataframe`` with the same key.

        Returns
        -------
        pandas.DataFrame
            A new DataFrame whose values are the decrypted strings (object dtype: numbers and missing values come back
            as text such as ``'2'`` and ``'nan'``) and whose column names no longer contain the suffix.

        Raises
        ------
        ValueError
            If a value cannot be decrypted, for example because the key is wrong or a column was never encrypted.

        Examples
        --------
        >>> import pandas as pd
        >>> df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})
        >>> encryptor = TextEncryptor(key="secret")
        >>> decrypted = encryptor.decrypt_dataframe(encryptor.encrypt_dataframe(df))
        >>> decrypted.columns.tolist()
        ['name', 'age']
        >>> decrypted['age'].tolist()
        ['25', '30']
        """
        res = data.copy()
        collist = data.columns.tolist()
        for col in collist:
            ## Encryption
            res[col] = res[col].apply(lambda x: self.decrypt(x))
        res.columns = [x.replace(self.suffix, "") for x in res.columns]
        return res

encrypt

encrypt(text)

Encrypt a string.

The UTF-8 bytes of text are XORed with the repeated key and prefixed with 2 bytes that hold the length of the plaintext in bytes, which decrypt uses for validation. The result is encoded as URL-safe Base64.

Parameters:

Name	Type	Description	Default
text	str	Plaintext to encrypt, at most 65,535 bytes of UTF-8.	required

Returns:

Type	Description
str	Encrypted text, URL-safe Base64.

Raises:

Type	Description
TypeError	If the key is None.
ZeroDivisionError	If the key is an empty string.
OverflowError	If text is longer than 65,535 bytes.

Examples:

>>> encryptor = TextEncryptor(key="secret")
>>> encryptor.encrypt("Hello")
'AAU7AA8eCg=='

Source code in Modeling_Tool/Core/XOR_Encryptor.py

def encrypt(self, text):
    """
    Encrypt a string.

    The UTF-8 bytes of ``text`` are XORed with the repeated key and prefixed with 2 bytes that hold the length of
    the plaintext in bytes, which ``decrypt`` uses for validation. The result is encoded as URL-safe Base64.

    Parameters
    ----------
    text : str
        Plaintext to encrypt, at most 65,535 bytes of UTF-8.

    Returns
    -------
    str
        Encrypted text, URL-safe Base64.

    Raises
    ------
    TypeError
        If the key is None.
    ZeroDivisionError
        If the key is an empty string.
    OverflowError
        If ``text`` is longer than 65,535 bytes.

    Examples
    --------
    >>> encryptor = TextEncryptor(key="secret")
    >>> encryptor.encrypt("Hello")
    'AAU7AA8eCg=='
    """
    # Text to bytes
    text_bytes = text.encode('utf-8')

    # Expand byte length
    key_bytes = (self.key * (len(text_bytes) // len(self.key) + 1)).encode('utf-8')
    key_bytes = key_bytes[:len(text_bytes)]

    # XOR encryption
    encrypted_bytes = bytes([text_bytes[i] ^ key_bytes[i] for i in range(len(text_bytes))])

    # Add 2 more bytes for verification: the plaintext length in bytes, which is what decrypt compares against
    length_byte = len(text_bytes).to_bytes(2, 'big')

    # combine
    result_bytes = length_byte + encrypted_bytes
    return base64.urlsafe_b64encode(result_bytes).decode('utf-8')

decrypt

decrypt(encrypted_text)

Decrypt a string produced by encrypt.

The Base64 text is decoded, the first 2 bytes give the stored plaintext length, and the remaining bytes are XORed with the repeated key. The length of the recovered bytes is checked against the stored length.

Parameters:

Name	Type	Description	Default
encrypted_text	str	URL-safe Base64 text produced by encrypt.	required

Returns:

Type	Description
str	The original plaintext.

Raises:

Type	Description
ValueError	If decryption fails for any reason: the input is not valid Base64, the key is wrong or missing, the stored length does not match the recovered length (the data was altered), or the recovered bytes are not valid UTF-8.

Examples:

>>> encryptor = TextEncryptor(key="secret")
>>> encryptor.decrypt(encryptor.encrypt("Hello"))
'Hello'

Source code in Modeling_Tool/Core/XOR_Encryptor.py

def decrypt(self, encrypted_text):
    """
    Decrypt a string produced by ``encrypt``.

    The Base64 text is decoded, the first 2 bytes give the stored plaintext length, and the remaining bytes are
    XORed with the repeated key. The length of the recovered bytes is checked against the stored length.

    Parameters
    ----------
    encrypted_text : str
        URL-safe Base64 text produced by ``encrypt``.

    Returns
    -------
    str
        The original plaintext.

    Raises
    ------
    ValueError
        If decryption fails for any reason: the input is not valid Base64, the key is wrong or missing, the stored
        length does not match the recovered length (the data was altered), or the recovered bytes are not valid
        UTF-8.

    Examples
    --------
    >>> encryptor = TextEncryptor(key="secret")
    >>> encryptor.decrypt(encryptor.encrypt("Hello"))
    'Hello'
    """
    try:
        # b64 decryption
        decoded_bytes = base64.urlsafe_b64decode(encrypted_text.encode('utf-8'))

        # extract length info
        text_length = int.from_bytes(decoded_bytes[:2], 'big')

        # extract encryption info
        encrypted_bytes = decoded_bytes[2:]

        # regenerate key
        key_bytes = (self.key * (len(encrypted_bytes) // len(self.key) + 1)).encode('utf-8')
        key_bytes = key_bytes[:len(encrypted_bytes)]

        # XOR Decrypt
        decrypted_bytes = bytes([encrypted_bytes[i] ^ key_bytes[i] for i in range(len(encrypted_bytes))])

        # Check Length
        if len(decrypted_bytes) != text_length:
            raise ValueError("Text Length Does Not Match!")

        return decrypted_bytes.decode('utf-8')
    except:
        raise ValueError("Decrypt Failed! Data Might be Destroyed or Incorrect Key!")

encrypt_dataframe

encrypt_dataframe(data)

Encrypt every value of a DataFrame.

All values are converted to strings (a missing value becomes 'nan') and encrypted with encrypt, and the suffix is appended to every column name. The original DataFrame is not modified.

Parameters:

Name	Type	Description	Default
data	DataFrame	The DataFrame to encrypt.	required

Returns:

Type	Description
DataFrame	A new DataFrame of encrypted strings (object dtype) whose column names carry the suffix.

Raises:

Type	Description
TypeError	If the key is None.
ZeroDivisionError	If the key is an empty string.
OverflowError	If a value, converted to a string, is longer than 65,535 bytes.

Notes

An encrypted DataFrame cannot be analyzed; decrypt it first. A column name that already contains the suffix text does not survive a round trip, because decrypt_dataframe removes every occurrence of the suffix.

Examples:

>>> import pandas as pd
>>> df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})
>>> encryptor = TextEncryptor(key="secret")
>>> encryptor.encrypt_dataframe(df).columns.tolist()
['name_encrypted', 'age_encrypted']

Source code in Modeling_Tool/Core/XOR_Encryptor.py

def encrypt_dataframe(self, data):
    """
    Encrypt every value of a DataFrame.

    All values are converted to strings (a missing value becomes ``'nan'``) and encrypted with ``encrypt``, and the
    suffix is appended to every column name. The original DataFrame is not modified.

    Parameters
    ----------
    data : pandas.DataFrame
        The DataFrame to encrypt.

    Returns
    -------
    pandas.DataFrame
        A new DataFrame of encrypted strings (object dtype) whose column names carry the suffix.

    Raises
    ------
    TypeError
        If the key is None.
    ZeroDivisionError
        If the key is an empty string.
    OverflowError
        If a value, converted to a string, is longer than 65,535 bytes.

    Notes
    -----
    An encrypted DataFrame cannot be analyzed; decrypt it first. A column name that already contains the suffix text
    does not survive a round trip, because ``decrypt_dataframe`` removes every occurrence of the suffix.

    Examples
    --------
    >>> import pandas as pd
    >>> df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})
    >>> encryptor = TextEncryptor(key="secret")
    >>> encryptor.encrypt_dataframe(df).columns.tolist()
    ['name_encrypted', 'age_encrypted']
    """
    res = data.copy()
    collist = data.columns.tolist()
    for col in collist:
        ## Encryption
        res[col] = res[col].astype(str)
        res[col] = res[col].apply(lambda x: self.encrypt(x))
    res.columns = [x + self.suffix for x in res.columns]
    return res

decrypt_dataframe

decrypt_dataframe(data)

Decrypt a DataFrame produced by encrypt_dataframe.

Every value is decrypted with decrypt and every occurrence of the suffix is removed from the column names. The original DataFrame is not modified.

Parameters:

Name	Type	Description	Default
data	DataFrame	A DataFrame produced by encrypt_dataframe with the same key.	required

Returns:

Type	Description
DataFrame	A new DataFrame whose values are the decrypted strings (object dtype: numbers and missing values come back as text such as '2' and 'nan') and whose column names no longer contain the suffix.

Raises:

Type	Description
ValueError	If a value cannot be decrypted, for example because the key is wrong or a column was never encrypted.

Examples:

>>> import pandas as pd
>>> df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})
>>> encryptor = TextEncryptor(key="secret")
>>> decrypted = encryptor.decrypt_dataframe(encryptor.encrypt_dataframe(df))
>>> decrypted.columns.tolist()
['name', 'age']
>>> decrypted['age'].tolist()
['25', '30']

Source code in Modeling_Tool/Core/XOR_Encryptor.py

def decrypt_dataframe(self, data):
    """
    Decrypt a DataFrame produced by ``encrypt_dataframe``.

    Every value is decrypted with ``decrypt`` and every occurrence of the suffix is removed from the column names.
    The original DataFrame is not modified.

    Parameters
    ----------
    data : pandas.DataFrame
        A DataFrame produced by ``encrypt_dataframe`` with the same key.

    Returns
    -------
    pandas.DataFrame
        A new DataFrame whose values are the decrypted strings (object dtype: numbers and missing values come back
        as text such as ``'2'`` and ``'nan'``) and whose column names no longer contain the suffix.

    Raises
    ------
    ValueError
        If a value cannot be decrypted, for example because the key is wrong or a column was never encrypted.

    Examples
    --------
    >>> import pandas as pd
    >>> df = pd.DataFrame({'name': ['Alice', 'Bob'], 'age': [25, 30]})
    >>> encryptor = TextEncryptor(key="secret")
    >>> decrypted = encryptor.decrypt_dataframe(encryptor.encrypt_dataframe(df))
    >>> decrypted.columns.tolist()
    ['name', 'age']
    >>> decrypted['age'].tolist()
    ['25', '30']
    """
    res = data.copy()
    collist = data.columns.tolist()
    for col in collist:
        ## Encryption
        res[col] = res[col].apply(lambda x: self.decrypt(x))
    res.columns = [x.replace(self.suffix, "") for x in res.columns]
    return res

Extended DataFrame: kDataFrame

kDataFrame

kSeries

Bases: Series

The extension class of Pandas Series, paired with kDataFrame: operations on it return kSeries / kDataFrame objects, and it adds a few scoring and frequency-table helpers.

Parameters:

Name	Type	Description	Default
data	array-like, Iterable, dict or scalar value	The data of the series, exactly as accepted by pandas.Series. The argument is required.	required
*args	tuple	Further positional arguments, forwarded unchanged to pandas.Series.	()
**kwargs	dict	Further keyword arguments (for example index, dtype, name and copy), forwarded unchanged to pandas.Series.	{}

Source code in Modeling_Tool/Core/kDataFrame.py

class kSeries(Series):
    """
    The extension class of Pandas Series, paired with ``kDataFrame``: operations on it return ``kSeries`` /
    ``kDataFrame`` objects, and it adds a few scoring and frequency-table helpers.

    Parameters
    ----------
    data : array-like, Iterable, dict or scalar value
        The data of the series, exactly as accepted by ``pandas.Series``. The argument is required.
    *args : tuple
        Further positional arguments, forwarded unchanged to ``pandas.Series``.
    **kwargs : dict
        Further keyword arguments (for example ``index``, ``dtype``, ``name`` and ``copy``), forwarded unchanged to
        ``pandas.Series``.
    """
    @property
    def _constructor(self):
        return kSeries

    @property
    def _constructor_expanddim(self):
        return kDataFrame

    def __init__(self, data, *args, **kwargs):
        super().__init__(data, *args, **kwargs)

    def to_pdSeries(self, inplace: bool = False):
        """
        Convert kSeries back to pandas Series.

        Parameters
        ----------
        inplace : bool, default False
            If False, convert a copy of the kSeries. If True, convert the kSeries itself without copying it first, so
            the returned Series shares its data with this object. The kSeries is never modified or replaced.

        Returns
        -------
        pandas.Series
            A plain ``pandas.Series`` (not a ``kSeries``) with the same values, index and name.
        """
        if inplace:
            data = self
        else:
            data = self.copy()

        df = Series(data)
        return df

    def odds_score(self):
        """
        Calculate the odd score based on the given probability score.

        Odds Score = (count of sth happening) / (count of sth not happening)
        Odds Score = pb_score / (1-pb_score)  [This ranges from 0 to infinity.]
        Log Odds Score = np.log(pb_score / (1-pb_score))  [This ranges from -infinity to +infinity]
        This is helpful for solving binary classification problem.
        Odds Ratio: The ratio of odds.

        The values of the series are used as ``pb_score``, the predicted probability of the event.

        Returns
        -------
        kSeries
            The scorecard-style score ``500 - 20 / ln(2) * (ln(15) + ln(pb_score / (1 - pb_score)))`` for every value,
            with the index and name of the series. It is not the raw odds: a higher probability gives a lower score,
            every doubling of the odds lowers the score by 20 points, and ``pb_score = 1/16`` (odds of 1 to 15) maps
            to exactly 500.

        Notes
        -----
        The probabilities are not validated. ``0`` gives ``inf``, ``1`` gives ``-inf``, and values outside ``[0, 1]``
        or missing values give NaN. For other base score, odds or points-to-double-the-odds settings use
        ``Modeling_Tool.Core.utils.odds_score``.
        """
        pb_score = self
        a = (20 / np.log(2))
        b = (np.log(15) + np.log(pb_score/(1 - pb_score)))
        return (500 - a * b)

    def scale_score(self):
        """
        Scale the model scores (for internt segment of MCI model)

        Returns
        -------
        kSeries
            The values multiplied by 1.112 and capped at 0.9999999 (the index and name of the series are kept).
        """
        return (self * 1.112).clip(upper=0.9999999)

    def proc_freq(self) -> pd.DataFrame:
        """
        Implement the Python version "proc freq" query in SAS.

        Returns
        -------
        pandas.DataFrame
            One row per distinct value of the series (missing values count as a row of their own), indexed by that
            value and sorted by decreasing frequency. The columns are ``frequency`` (count), ``percent`` (share of all
            values), ``cumFrequency`` and ``cumPercent`` (running sums of the first two, in the row order).
        """
        data = self.copy()
        f = data.value_counts(dropna = False)
        p = data.value_counts(dropna = False, normalize = True)
        df = pd.concat([f,p], axis = 1, keys = ['frequency', 'percent'])
        df['cumFrequency'] = df['frequency'].cumsum()
        df['cumPercent'] = df['percent'].cumsum()
        return df

to_pdSeries

to_pdSeries(inplace: bool = False)

Convert kSeries back to pandas Series.

Parameters:

Name	Type	Description	Default
inplace	bool	If False, convert a copy of the kSeries. If True, convert the kSeries itself without copying it first, so the returned Series shares its data with this object. The kSeries is never modified or replaced.	False

Returns:

Type	Description
Series	A plain pandas.Series (not a kSeries) with the same values, index and name.

Source code in Modeling_Tool/Core/kDataFrame.py

def to_pdSeries(self, inplace: bool = False):
    """
    Convert kSeries back to pandas Series.

    Parameters
    ----------
    inplace : bool, default False
        If False, convert a copy of the kSeries. If True, convert the kSeries itself without copying it first, so
        the returned Series shares its data with this object. The kSeries is never modified or replaced.

    Returns
    -------
    pandas.Series
        A plain ``pandas.Series`` (not a ``kSeries``) with the same values, index and name.
    """
    if inplace:
        data = self
    else:
        data = self.copy()

    df = Series(data)
    return df

odds_score

odds_score()

Calculate the odd score based on the given probability score.

Odds Score = (count of sth happening) / (count of sth not happening) Odds Score = pb_score / (1-pb_score) [This ranges from 0 to infinity.] Log Odds Score = np.log(pb_score / (1-pb_score)) [This ranges from -infinity to +infinity] This is helpful for solving binary classification problem. Odds Ratio: The ratio of odds.

The values of the series are used as pb_score, the predicted probability of the event.

Returns:

Type	Description
kSeries	The scorecard-style score 500 - 20 / ln(2) * (ln(15) + ln(pb_score / (1 - pb_score))) for every value, with the index and name of the series. It is not the raw odds: a higher probability gives a lower score, every doubling of the odds lowers the score by 20 points, and pb_score = 1/16 (odds of 1 to 15) maps to exactly 500.

Notes

The probabilities are not validated. 0 gives inf, 1 gives -inf, and values outside [0, 1] or missing values give NaN. For other base score, odds or points-to-double-the-odds settings use Modeling_Tool.Core.utils.odds_score.

Source code in Modeling_Tool/Core/kDataFrame.py

def odds_score(self):
    """
    Calculate the odd score based on the given probability score.

    Odds Score = (count of sth happening) / (count of sth not happening)
    Odds Score = pb_score / (1-pb_score)  [This ranges from 0 to infinity.]
    Log Odds Score = np.log(pb_score / (1-pb_score))  [This ranges from -infinity to +infinity]
    This is helpful for solving binary classification problem.
    Odds Ratio: The ratio of odds.

    The values of the series are used as ``pb_score``, the predicted probability of the event.

    Returns
    -------
    kSeries
        The scorecard-style score ``500 - 20 / ln(2) * (ln(15) + ln(pb_score / (1 - pb_score)))`` for every value,
        with the index and name of the series. It is not the raw odds: a higher probability gives a lower score,
        every doubling of the odds lowers the score by 20 points, and ``pb_score = 1/16`` (odds of 1 to 15) maps
        to exactly 500.

    Notes
    -----
    The probabilities are not validated. ``0`` gives ``inf``, ``1`` gives ``-inf``, and values outside ``[0, 1]``
    or missing values give NaN. For other base score, odds or points-to-double-the-odds settings use
    ``Modeling_Tool.Core.utils.odds_score``.
    """
    pb_score = self
    a = (20 / np.log(2))
    b = (np.log(15) + np.log(pb_score/(1 - pb_score)))
    return (500 - a * b)

scale_score

scale_score()

Scale the model scores (for internt segment of MCI model)

Returns:

Type	Description
kSeries	The values multiplied by 1.112 and capped at 0.9999999 (the index and name of the series are kept).

Source code in Modeling_Tool/Core/kDataFrame.py

def scale_score(self):
    """
    Scale the model scores (for internt segment of MCI model)

    Returns
    -------
    kSeries
        The values multiplied by 1.112 and capped at 0.9999999 (the index and name of the series are kept).
    """
    return (self * 1.112).clip(upper=0.9999999)

proc_freq

proc_freq() -> DataFrame

Implement the Python version "proc freq" query in SAS.

Returns:

Type	Description
DataFrame	One row per distinct value of the series (missing values count as a row of their own), indexed by that value and sorted by decreasing frequency. The columns are frequency (count), percent (share of all values), cumFrequency and cumPercent (running sums of the first two, in the row order).

Source code in Modeling_Tool/Core/kDataFrame.py

def proc_freq(self) -> pd.DataFrame:
    """
    Implement the Python version "proc freq" query in SAS.

    Returns
    -------
    pandas.DataFrame
        One row per distinct value of the series (missing values count as a row of their own), indexed by that
        value and sorted by decreasing frequency. The columns are ``frequency`` (count), ``percent`` (share of all
        values), ``cumFrequency`` and ``cumPercent`` (running sums of the first two, in the row order).
    """
    data = self.copy()
    f = data.value_counts(dropna = False)
    p = data.value_counts(dropna = False, normalize = True)
    df = pd.concat([f,p], axis = 1, keys = ['frequency', 'percent'])
    df['cumFrequency'] = df['frequency'].cumsum()
    df['cumPercent'] = df['percent'].cumsum()
    return df

kDataFrame

Bases: DataFrame

The extension class of Pandas DataFrame including more useful methods to manipulate dataset.

Parameters:

Name	Type	Description	Default
data	DataFrame	Any pandas dataframe dataset. Anything accepted by pandas.DataFrame (dict, list of records, ndarray, DataFrame, ...) works, but unlike pandas the argument is required.	required
*args	tuple	Further positional arguments, forwarded unchanged to pandas.DataFrame.	()
**kwargs	dict	Further keyword arguments (for example index, columns, dtype and copy), forwarded unchanged to pandas.DataFrame.	{}

Returns:

Type	Description
kDataFrame	The extension object of dataframe.

Attributes:

Name	Type	Description
added_property	int, default 1	Example of a metadata attribute: it is listed in _metadata, so pandas passes it on to copies. SMF does not use it anywhere else.

Source code in Modeling_Tool/Core/kDataFrame.py

class kDataFrame(DataFrame):
    """
    The extension class of Pandas DataFrame including more useful methods to manipulate dataset.

    Parameters
    ----------
    data : DataFrame
        Any pandas dataframe dataset. Anything accepted by ``pandas.DataFrame`` (dict, list of records, ndarray,
        DataFrame, ...) works, but unlike pandas the argument is required.
    *args : tuple
        Further positional arguments, forwarded unchanged to ``pandas.DataFrame``.
    **kwargs : dict
        Further keyword arguments (for example ``index``, ``columns``, ``dtype`` and ``copy``), forwarded unchanged to
        ``pandas.DataFrame``.

    Returns
    -------
    kDataFrame
        The extension object of dataframe.

    Attributes
    ----------
    added_property : int, default 1
        Example of a metadata attribute: it is listed in ``_metadata``, so pandas passes it on to copies. SMF does
        not use it anywhere else.
    """
    _metadata = ['added_property']
    added_property = 1  # This will be passed to copies

    @property
    def _constructor(self):
        return kDataFrame

    @property
    def _constructor_sliced(self):
        return kSeries

    def __init__(self, data, *args, **kwargs):
        super().__init__(data, *args, **kwargs)

    def move_column(self, colname: str, idx: int, inplace: bool = False):
        """
        To move a column into specific place by index.

        Parameters
        ----------
        colname : str
            Name of the column to move. It must be a column of the frame.
        idx : int
            Target position (0-based) of the column, with ``list.insert`` semantics: 0 puts it first and a value
            at or beyond the number of columns puts it last.
        inplace : bool, default False
            If False, work on a copy; if True, work on the frame itself. This does not change the result: the
            reordered frame is built with ``data[colarray]``, a new object in both cases, so the frame the method is
            called on is never reordered (see Notes).

        Returns
        -------
        kDataFrame
            A new frame with the same data and the columns reordered.

        Notes
        -----
        ``inplace=True`` is effectively a no-op for the original frame: always use the returned frame.
        """
        if inplace:
            data = self
        else:
            data = self.copy()
        colarray = data.columns.tolist()
        colarray.remove(colname)
        colarray.insert(idx, colname)
        data = data[colarray]
        return data

    def convert_to_vintage(self, vintage_colname: str = 'VINTAGE', by: str = 'TRAN_TMS', inplace: bool = False):
        """
        To obtain a vintage column by a time/data column.

        Parameters
        ----------
        vintage_colname : str, default 'VINTAGE'
            Name of the column that receives the vintage. A column with this name is overwritten.
        by : str, default 'TRAN_TMS'
            Name of the date/time column the vintage is taken from. The column is cast to string and the first
            ``YYYY-MM`` pattern in the text is used (for example from ``2024-05-17`` or ``2024-05-17 10:30:00``).
        inplace : bool, default False
            If True, add the column to the frame itself; if False, add it to a copy and leave the frame unchanged.

        Returns
        -------
        kDataFrame
            The frame holding the new column: the frame itself when ``inplace=True``, otherwise a copy. The column
            has pandas ``string`` dtype and holds the vintage as ``YYYYMM`` text (``'202405'``).

        Notes
        -----
        Values that contain no ``YYYY-MM`` pattern (other separators such as ``2024/05/17``, compact dates such as
        ``20240517``, or missing values) become missing (``<NA>``) in the vintage column.
        """
        if inplace:
            data = self
        else:
            data = self.copy()
        data[vintage_colname] = (
            data[by]
            .astype("string")
            .str.extract(r"(\d{4}-\d{2})", expand=False)
            .str.replace("-", "", regex=False)
        )
        if inplace:
            self = data
        return data

    def col_filter_regex(self, regex: str = ".*?of_co_at_12m", case_sensitive = True, inplace: bool = False):
        """
        To filter the DataFrame columns by regular expression.

        Parameters
        ----------
        regex : str, default ".*?of_co_at_12m"
            Regular expression searched (``re.search``, so it may match anywhere) in every column name; the columns
            whose name contains a match are kept.
        case_sensitive : bool, default True
            If False, the match ignores case.
        inplace : bool, default False
            If False, work on a copy; if True, work on the frame itself. This does not change the result: the frame
            the method is called on is never modified in either case.

        Returns
        -------
        kDataFrame
            A new frame holding only the matching columns, in their original order (a frame without columns when
            nothing matches).

        Notes
        -----
        The column names must be strings, because the pattern is applied through ``DataFrame.columns.str``.
        """
        if inplace:
            data = self
        else:
            data = self.copy()

        fltr = data.columns[data.columns.str.contains(regex, regex = True, case = case_sensitive)]
        return data[fltr]

    def row_filter_regex(self, col = None, regex: str = None, case_sensitive = True,
                         as_index = False, inplace: bool = False):
        """
        To filter the string format row using regex.

        Parameters
        ----------
        col : str, default None
            Name of the column whose values are tested. Required in practice: ``None`` raises ``KeyError``.
        regex : str, default None
            Regular expression searched (``re.search``, so it may match anywhere) in the text of every value.
            Required in practice: ``None`` raises ``TypeError``.
        case_sensitive : bool, default True
            If False, the match ignores case.
        as_index : bool, default False
            If True, the returned rows are indexed by ``col`` (``set_index(col)``), so the column is moved out of the
            regular columns.
        inplace : bool, default False
            If False, work on a copy; if True, work on the frame itself. This does not change the result: the frame
            the method is called on is never modified in either case.

        Returns
        -------
        kDataFrame
            A new frame with the rows whose value in ``col`` matches ``regex``, keeping their original index (or
            indexed by ``col`` when ``as_index=True``).

        Notes
        -----
        The values of ``col`` are cast to ``str`` before matching, so numbers are matched through their text
        (``'1.5'``) and missing values through ``'nan'`` or ``'None'``.
        """
        if inplace:
            data = self
        else:
            data = self.copy()

        fltr = data[col].astype('str').str.contains(pat = regex, regex = True, case = case_sensitive)
        if as_index:
            return data[fltr].set_index(col)
        return data[fltr]

    def scale_score(self, pb_score: str):
        """
        Scale the model scores (for internt segment of MCI model)

        Parameters
        ----------
        pb_score : str
            Name of the column that holds the model score (a probability).

        Returns
        -------
        kSeries
            The column multiplied by 1.112 and capped at 0.9999999, with the index of the frame. The frame itself
            is not modified (no column is added).
        """
        return (self[pb_score] * 1.112).clip(upper=0.9999999)

    def proc_freq(self, var: str):
        """
        Implement the Python version "proc freq" query in SAS.

        Parameters
        ----------
        var : str
            Name of the column to tabulate.

        Returns
        -------
        pandas.DataFrame
            One row per distinct value of the column (missing values count as a row of their own), indexed by that
            value and sorted by decreasing frequency. The columns are ``frequency`` (count), ``percent`` (share of all
            rows), ``cumFrequency`` and ``cumPercent`` (running sums of the first two, in the row order).
        """
        data = self.copy()
        f = data[var].value_counts(dropna = False)
        p = data[var].value_counts(dropna = False, normalize = True)
        df = pd.concat([f,p], axis = 1, keys = ['frequency', 'percent'])
        df['cumFrequency'] = df['frequency'].cumsum()
        df['cumPercent'] = df['percent'].cumsum()
        return df

    def unify_table_col_names(self, how: str = "lowercase", inplace: bool = False):
        """
        Unify the format of column names.

        Parameters
        ----------
        how : str, default "lowercase"
            Target format, compared case-insensitively: ``"lower"`` or ``"lowercase"``, ``"upper"`` or
            ``"uppercase"``, ``"cap"`` or ``"capitalize"`` (first letter upper case, the rest lower case).
        inplace : bool, default False
            If True, rename the columns of the frame itself; if False, work on a copy and leave the frame unchanged.

        Returns
        -------
        kDataFrame
            The frame with the renamed columns: the frame itself when ``inplace=True``, otherwise a copy.

        Notes
        -----
        The column names must be strings. ``how`` is not validated: any other value fails with
        ``UnboundLocalError``. Names that collide after the conversion (``'A'`` and ``'a'``) become duplicate
        column names.
        """

        if inplace:
            data = self
        else:
            data = self.copy()

        cols = data.columns
        if how.lower() == "lower" or how.lower() == "lowercase":
            res = [name.lower() for name in cols]
        if how.lower() == "upper" or how.lower() == "uppercase":
            res = [name.upper() for name in cols]
        if how.lower() == "cap" or how.lower() == "capitalize":
            res = [name.capitalize() for name in cols]
        data.columns = res
        return data

    def convert_strlist_to_list(self, col: str):
        """
        cast string-type lists in a specified Series into real lists.

        Parameters
        ----------
        col : str
            Name of the column that holds the list-like strings, for example ``"['a', 'b']"``.

        Returns
        -------
        kSeries
            One list of strings per row: the runs of letters, digits and underscores found in ``str(value)``, so
            quotes, brackets and commas are dropped (``"['a', 'b']"`` becomes ``['a', 'b']``).

        Notes
        -----
        The series is rebuilt from the extracted lists, so it has a new default index (0 to n-1) and not the index
        of the frame. Any other punctuation also splits a token (``'1.5'`` becomes ``['1', '5']``), and a missing
        value becomes ``['nan']`` or ``['None']``.
        """
        import re

        data = self.copy()
        str_col = kSeries([re.findall(r"\w+", str(x)) for x in data[col]])
        return str_col

    def to_pdDataFrame(self, inplace: bool = False):
        """
        Convert df_extension back to DataFrame.

        Parameters
        ----------
        inplace : bool, default False
            If False, convert a copy of the kDataFrame. If True, convert the kDataFrame itself without copying it
            first, so the returned DataFrame shares its data with this object. The kDataFrame is never modified or
            replaced.

        Returns
        -------
        pandas.DataFrame
            A plain ``pandas.DataFrame`` (not a ``kDataFrame``) with the same data, index and columns.
        """
        if inplace:
            data = self
        else:
            data = self.copy()

        df = DataFrame(data)
        return df

move_column

move_column(colname: str, idx: int, inplace: bool = False)

To move a column into specific place by index.

Parameters:

Name	Type	Description	Default
colname	str	Name of the column to move. It must be a column of the frame.	required
idx	int	Target position (0-based) of the column, with list.insert semantics: 0 puts it first and a value at or beyond the number of columns puts it last.	required
inplace	bool	If False, work on a copy; if True, work on the frame itself. This does not change the result: the reordered frame is built with data[colarray], a new object in both cases, so the frame the method is called on is never reordered (see Notes).	False

Returns:

Type	Description
kDataFrame	A new frame with the same data and the columns reordered.

Notes

inplace=True is effectively a no-op for the original frame: always use the returned frame.

Source code in Modeling_Tool/Core/kDataFrame.py

def move_column(self, colname: str, idx: int, inplace: bool = False):
    """
    To move a column into specific place by index.

    Parameters
    ----------
    colname : str
        Name of the column to move. It must be a column of the frame.
    idx : int
        Target position (0-based) of the column, with ``list.insert`` semantics: 0 puts it first and a value
        at or beyond the number of columns puts it last.
    inplace : bool, default False
        If False, work on a copy; if True, work on the frame itself. This does not change the result: the
        reordered frame is built with ``data[colarray]``, a new object in both cases, so the frame the method is
        called on is never reordered (see Notes).

    Returns
    -------
    kDataFrame
        A new frame with the same data and the columns reordered.

    Notes
    -----
    ``inplace=True`` is effectively a no-op for the original frame: always use the returned frame.
    """
    if inplace:
        data = self
    else:
        data = self.copy()
    colarray = data.columns.tolist()
    colarray.remove(colname)
    colarray.insert(idx, colname)
    data = data[colarray]
    return data

convert_to_vintage

convert_to_vintage(vintage_colname: str = 'VINTAGE', by: str = 'TRAN_TMS', inplace: bool = False)

To obtain a vintage column by a time/data column.

Parameters:

Name	Type	Description	Default
vintage_colname	str	Name of the column that receives the vintage. A column with this name is overwritten.	'VINTAGE'
by	str	Name of the date/time column the vintage is taken from. The column is cast to string and the first YYYY-MM pattern in the text is used (for example from 2024-05-17 or 2024-05-17 10:30:00).	'TRAN_TMS'
inplace	bool	If True, add the column to the frame itself; if False, add it to a copy and leave the frame unchanged.	False

Returns:

Type	Description
kDataFrame	The frame holding the new column: the frame itself when inplace=True, otherwise a copy. The column has pandas string dtype and holds the vintage as YYYYMM text ('202405').

Notes

Values that contain no YYYY-MM pattern (other separators such as 2024/05/17, compact dates such as 20240517, or missing values) become missing (<NA>) in the vintage column.

Source code in Modeling_Tool/Core/kDataFrame.py

def convert_to_vintage(self, vintage_colname: str = 'VINTAGE', by: str = 'TRAN_TMS', inplace: bool = False):
    """
    To obtain a vintage column by a time/data column.

    Parameters
    ----------
    vintage_colname : str, default 'VINTAGE'
        Name of the column that receives the vintage. A column with this name is overwritten.
    by : str, default 'TRAN_TMS'
        Name of the date/time column the vintage is taken from. The column is cast to string and the first
        ``YYYY-MM`` pattern in the text is used (for example from ``2024-05-17`` or ``2024-05-17 10:30:00``).
    inplace : bool, default False
        If True, add the column to the frame itself; if False, add it to a copy and leave the frame unchanged.

    Returns
    -------
    kDataFrame
        The frame holding the new column: the frame itself when ``inplace=True``, otherwise a copy. The column
        has pandas ``string`` dtype and holds the vintage as ``YYYYMM`` text (``'202405'``).

    Notes
    -----
    Values that contain no ``YYYY-MM`` pattern (other separators such as ``2024/05/17``, compact dates such as
    ``20240517``, or missing values) become missing (``<NA>``) in the vintage column.
    """
    if inplace:
        data = self
    else:
        data = self.copy()
    data[vintage_colname] = (
        data[by]
        .astype("string")
        .str.extract(r"(\d{4}-\d{2})", expand=False)
        .str.replace("-", "", regex=False)
    )
    if inplace:
        self = data
    return data

col_filter_regex

col_filter_regex(regex: str = '.*?of_co_at_12m', case_sensitive=True, inplace: bool = False)

To filter the DataFrame columns by regular expression.

Parameters:

Name	Type	Description	Default
regex	str	Regular expression searched (re.search, so it may match anywhere) in every column name; the columns whose name contains a match are kept.	".*?of_co_at_12m"
case_sensitive	bool	If False, the match ignores case.	True
inplace	bool	If False, work on a copy; if True, work on the frame itself. This does not change the result: the frame the method is called on is never modified in either case.	False

Returns:

Type	Description
kDataFrame	A new frame holding only the matching columns, in their original order (a frame without columns when nothing matches).

Notes

The column names must be strings, because the pattern is applied through DataFrame.columns.str.

Source code in Modeling_Tool/Core/kDataFrame.py

def col_filter_regex(self, regex: str = ".*?of_co_at_12m", case_sensitive = True, inplace: bool = False):
    """
    To filter the DataFrame columns by regular expression.

    Parameters
    ----------
    regex : str, default ".*?of_co_at_12m"
        Regular expression searched (``re.search``, so it may match anywhere) in every column name; the columns
        whose name contains a match are kept.
    case_sensitive : bool, default True
        If False, the match ignores case.
    inplace : bool, default False
        If False, work on a copy; if True, work on the frame itself. This does not change the result: the frame
        the method is called on is never modified in either case.

    Returns
    -------
    kDataFrame
        A new frame holding only the matching columns, in their original order (a frame without columns when
        nothing matches).

    Notes
    -----
    The column names must be strings, because the pattern is applied through ``DataFrame.columns.str``.
    """
    if inplace:
        data = self
    else:
        data = self.copy()

    fltr = data.columns[data.columns.str.contains(regex, regex = True, case = case_sensitive)]
    return data[fltr]

row_filter_regex

row_filter_regex(col=None, regex: str = None, case_sensitive=True, as_index=False, inplace: bool = False)

To filter the string format row using regex.

Parameters:

Name	Type	Description	Default
col	str	Name of the column whose values are tested. Required in practice: None raises KeyError.	None
regex	str	Regular expression searched (re.search, so it may match anywhere) in the text of every value. Required in practice: None raises TypeError.	None
case_sensitive	bool	If False, the match ignores case.	True
as_index	bool	If True, the returned rows are indexed by col (set_index(col)), so the column is moved out of the regular columns.	False
inplace	bool	If False, work on a copy; if True, work on the frame itself. This does not change the result: the frame the method is called on is never modified in either case.	False

Returns:

Type	Description
kDataFrame	A new frame with the rows whose value in col matches regex, keeping their original index (or indexed by col when as_index=True).

Notes

The values of col are cast to str before matching, so numbers are matched through their text ('1.5') and missing values through 'nan' or 'None'.

Source code in Modeling_Tool/Core/kDataFrame.py

def row_filter_regex(self, col = None, regex: str = None, case_sensitive = True,
                     as_index = False, inplace: bool = False):
    """
    To filter the string format row using regex.

    Parameters
    ----------
    col : str, default None
        Name of the column whose values are tested. Required in practice: ``None`` raises ``KeyError``.
    regex : str, default None
        Regular expression searched (``re.search``, so it may match anywhere) in the text of every value.
        Required in practice: ``None`` raises ``TypeError``.
    case_sensitive : bool, default True
        If False, the match ignores case.
    as_index : bool, default False
        If True, the returned rows are indexed by ``col`` (``set_index(col)``), so the column is moved out of the
        regular columns.
    inplace : bool, default False
        If False, work on a copy; if True, work on the frame itself. This does not change the result: the frame
        the method is called on is never modified in either case.

    Returns
    -------
    kDataFrame
        A new frame with the rows whose value in ``col`` matches ``regex``, keeping their original index (or
        indexed by ``col`` when ``as_index=True``).

    Notes
    -----
    The values of ``col`` are cast to ``str`` before matching, so numbers are matched through their text
    (``'1.5'``) and missing values through ``'nan'`` or ``'None'``.
    """
    if inplace:
        data = self
    else:
        data = self.copy()

    fltr = data[col].astype('str').str.contains(pat = regex, regex = True, case = case_sensitive)
    if as_index:
        return data[fltr].set_index(col)
    return data[fltr]

scale_score

scale_score(pb_score: str)

Scale the model scores (for internt segment of MCI model)

Parameters:

Name	Type	Description	Default
pb_score	str	Name of the column that holds the model score (a probability).	required

Returns:

Type	Description
kSeries	The column multiplied by 1.112 and capped at 0.9999999, with the index of the frame. The frame itself is not modified (no column is added).

Source code in Modeling_Tool/Core/kDataFrame.py

def scale_score(self, pb_score: str):
    """
    Scale the model scores (for internt segment of MCI model)

    Parameters
    ----------
    pb_score : str
        Name of the column that holds the model score (a probability).

    Returns
    -------
    kSeries
        The column multiplied by 1.112 and capped at 0.9999999, with the index of the frame. The frame itself
        is not modified (no column is added).
    """
    return (self[pb_score] * 1.112).clip(upper=0.9999999)

proc_freq

proc_freq(var: str)

Implement the Python version "proc freq" query in SAS.

Parameters:

Name	Type	Description	Default
var	str	Name of the column to tabulate.	required

Returns:

Type	Description
DataFrame	One row per distinct value of the column (missing values count as a row of their own), indexed by that value and sorted by decreasing frequency. The columns are frequency (count), percent (share of all rows), cumFrequency and cumPercent (running sums of the first two, in the row order).

Source code in Modeling_Tool/Core/kDataFrame.py

def proc_freq(self, var: str):
    """
    Implement the Python version "proc freq" query in SAS.

    Parameters
    ----------
    var : str
        Name of the column to tabulate.

    Returns
    -------
    pandas.DataFrame
        One row per distinct value of the column (missing values count as a row of their own), indexed by that
        value and sorted by decreasing frequency. The columns are ``frequency`` (count), ``percent`` (share of all
        rows), ``cumFrequency`` and ``cumPercent`` (running sums of the first two, in the row order).
    """
    data = self.copy()
    f = data[var].value_counts(dropna = False)
    p = data[var].value_counts(dropna = False, normalize = True)
    df = pd.concat([f,p], axis = 1, keys = ['frequency', 'percent'])
    df['cumFrequency'] = df['frequency'].cumsum()
    df['cumPercent'] = df['percent'].cumsum()
    return df

unify_table_col_names

unify_table_col_names(how: str = 'lowercase', inplace: bool = False)

Unify the format of column names.

Parameters:

Name	Type	Description	Default
how	str	Target format, compared case-insensitively: "lower" or "lowercase", "upper" or "uppercase", "cap" or "capitalize" (first letter upper case, the rest lower case).	"lowercase"
inplace	bool	If True, rename the columns of the frame itself; if False, work on a copy and leave the frame unchanged.	False

Returns:

Type	Description
kDataFrame	The frame with the renamed columns: the frame itself when inplace=True, otherwise a copy.

Notes

The column names must be strings. how is not validated: any other value fails with UnboundLocalError. Names that collide after the conversion ('A' and 'a') become duplicate column names.

Source code in Modeling_Tool/Core/kDataFrame.py

def unify_table_col_names(self, how: str = "lowercase", inplace: bool = False):
    """
    Unify the format of column names.

    Parameters
    ----------
    how : str, default "lowercase"
        Target format, compared case-insensitively: ``"lower"`` or ``"lowercase"``, ``"upper"`` or
        ``"uppercase"``, ``"cap"`` or ``"capitalize"`` (first letter upper case, the rest lower case).
    inplace : bool, default False
        If True, rename the columns of the frame itself; if False, work on a copy and leave the frame unchanged.

    Returns
    -------
    kDataFrame
        The frame with the renamed columns: the frame itself when ``inplace=True``, otherwise a copy.

    Notes
    -----
    The column names must be strings. ``how`` is not validated: any other value fails with
    ``UnboundLocalError``. Names that collide after the conversion (``'A'`` and ``'a'``) become duplicate
    column names.
    """

    if inplace:
        data = self
    else:
        data = self.copy()

    cols = data.columns
    if how.lower() == "lower" or how.lower() == "lowercase":
        res = [name.lower() for name in cols]
    if how.lower() == "upper" or how.lower() == "uppercase":
        res = [name.upper() for name in cols]
    if how.lower() == "cap" or how.lower() == "capitalize":
        res = [name.capitalize() for name in cols]
    data.columns = res
    return data

convert_strlist_to_list

convert_strlist_to_list(col: str)

cast string-type lists in a specified Series into real lists.

Parameters:

Name	Type	Description	Default
col	str	Name of the column that holds the list-like strings, for example "['a', 'b']".	required

Returns:

Type	Description
kSeries	One list of strings per row: the runs of letters, digits and underscores found in str(value), so quotes, brackets and commas are dropped ("['a', 'b']" becomes ['a', 'b']).

Notes

The series is rebuilt from the extracted lists, so it has a new default index (0 to n-1) and not the index of the frame. Any other punctuation also splits a token ('1.5' becomes ['1', '5']), and a missing value becomes ['nan'] or ['None'].

Source code in Modeling_Tool/Core/kDataFrame.py

def convert_strlist_to_list(self, col: str):
    """
    cast string-type lists in a specified Series into real lists.

    Parameters
    ----------
    col : str
        Name of the column that holds the list-like strings, for example ``"['a', 'b']"``.

    Returns
    -------
    kSeries
        One list of strings per row: the runs of letters, digits and underscores found in ``str(value)``, so
        quotes, brackets and commas are dropped (``"['a', 'b']"`` becomes ``['a', 'b']``).

    Notes
    -----
    The series is rebuilt from the extracted lists, so it has a new default index (0 to n-1) and not the index
    of the frame. Any other punctuation also splits a token (``'1.5'`` becomes ``['1', '5']``), and a missing
    value becomes ``['nan']`` or ``['None']``.
    """
    import re

    data = self.copy()
    str_col = kSeries([re.findall(r"\w+", str(x)) for x in data[col]])
    return str_col

to_pdDataFrame

to_pdDataFrame(inplace: bool = False)

Convert df_extension back to DataFrame.

Parameters:

Name	Type	Description	Default
inplace	bool	If False, convert a copy of the kDataFrame. If True, convert the kDataFrame itself without copying it first, so the returned DataFrame shares its data with this object. The kDataFrame is never modified or replaced.	False

Returns:

Type	Description
DataFrame	A plain pandas.DataFrame (not a kDataFrame) with the same data, index and columns.

Source code in Modeling_Tool/Core/kDataFrame.py

def to_pdDataFrame(self, inplace: bool = False):
    """
    Convert df_extension back to DataFrame.

    Parameters
    ----------
    inplace : bool, default False
        If False, convert a copy of the kDataFrame. If True, convert the kDataFrame itself without copying it
        first, so the returned DataFrame shares its data with this object. The kDataFrame is never modified or
        replaced.

    Returns
    -------
    pandas.DataFrame
        A plain ``pandas.DataFrame`` (not a ``kDataFrame``) with the same data, index and columns.
    """
    if inplace:
        data = self
    else:
        data = self.copy()

    df = DataFrame(data)
    return df

DataFrame and JSON Conversion: Json_Data_Converter

Json_Data_Converter

df_to_json

df_to_json(drv_df: DataFrame, input_vars: Optional[List[str]] = None, metadata_cols: Optional[List[str]] = None) -> Dict[str, Any]

Convert a drv_df DataFrame to the JSON format.

Two modes: - Partitioned mode (input_vars specified): Metadata columns (the columns that are not in input_vars) are extracted as scalars, and the input_vars columns are collected into arrays placed under cdc_credit_inputs. Output: {"": , ..., "cdc_credit_inputs": {"": [...], ...}}

• Flat mode (input_vars=None): All columns are placed as arrays at the top level of the JSON, with no distinction between metadata / input_vars. Output: {"": [...], "": [...], ...}

Parameters:

Name	Type	Description	Default
drv_df	DataFrame	Source DataFrame, one record per row.	required
input_vars	Optional[List[str]]	List of model input feature column names. If None, flat mode is used and every column becomes an array.	None
metadata_cols	Optional[List[str]]	Metadata columns to specify explicitly in partitioned mode. Ignored in flat mode.	None

Returns:

Type	Description
Dict[str, Any]	Dictionary in JSON format.

Raises:

Type	Description
ValueError	In partitioned mode, if the metadata column values are inconsistent.
KeyError	If a column in input_vars does not exist in the DataFrame.

Source code in Modeling_Tool/Core/Json_Data_Converter.py

def df_to_json(
    drv_df: pd.DataFrame,
    input_vars: Optional[List[str]] = None,
    metadata_cols: Optional[List[str]] = None,
) -> Dict[str, Any]:
    """Convert a drv_df DataFrame to the JSON format.

    Two modes:
      - Partitioned mode (input_vars specified):
          Metadata columns (the columns that are not in input_vars) are extracted as scalars, and the input_vars
          columns are collected into arrays placed under cdc_credit_inputs.
          Output: {"<meta>": <scalar>, ..., "cdc_credit_inputs": {"<var>": [...], ...}}

      - Flat mode (input_vars=None):
          All columns are placed as arrays at the top level of the JSON, with no distinction between metadata / input_vars.
          Output: {"<col_1>": [...], "<col_2>": [...], ...}

    Parameters
    ----------
    drv_df : pd.DataFrame
        Source DataFrame, one record per row.
    input_vars : Optional[List[str]]
        List of model input feature column names. If None, flat mode is used and every column becomes an array.
    metadata_cols : Optional[List[str]]
        Metadata columns to specify explicitly in partitioned mode. Ignored in flat mode.

    Returns
    -------
    Dict[str, Any]
        Dictionary in JSON format.

    Raises
    ------
    ValueError
        In partitioned mode, if the metadata column values are inconsistent.
    KeyError
        If a column in input_vars does not exist in the DataFrame.
    """
    # ── Flat mode: all columns become top-level fields directly ──
    #   - Single-row DataFrame → values are scalars
    #   - Multi-row DataFrame → values are arrays
    if input_vars is None:
        result: Dict[str, Any] = {}
        if len(drv_df) == 1:
            first_row = drv_df.iloc[0]
            for col in drv_df.columns:
                result[col] = _safe_json_value(first_row[col])
        else:
            for col in drv_df.columns:
                result[col] = _safe_series_to_list(drv_df[col])
        return result

    # ── Partitioned mode (original logic) ──
    missing_cols = set(input_vars) - set(drv_df.columns)
    if missing_cols:
        raise KeyError(
            f"Columns in input_vars do not exist in the DataFrame: {missing_cols}"
        )

    if metadata_cols is None:
        metadata_cols = [c for c in drv_df.columns if c not in input_vars]

    missing_meta = set(metadata_cols) - set(drv_df.columns)
    if missing_meta:
        raise KeyError(
            f"Columns in metadata_cols do not exist in the DataFrame: {missing_meta}"
        )

    # Check that the metadata column values are consistent across the whole DataFrame
    for col in metadata_cols:
        unique_vals = drv_df[col].drop_duplicates()
        if len(unique_vals) > 1:
            raise ValueError(
                f"Metadata column '{col}' has multiple distinct values: {unique_vals.to_list()}. "
                f"Metadata columns are expected to be constant across all rows; check the data or adjust the metadata_cols argument."
            )

    result = {}

    # 1) Metadata: taken from the first row
    first_row = drv_df.iloc[0]
    for col in metadata_cols:
        result[col] = _safe_json_value(first_row[col])

    # 2) cdc_credit_inputs: collect the values of each input_var into an array
    cdc_credit_inputs: Dict[str, List[Any]] = {}
    for col in input_vars:
        cdc_credit_inputs[col] = _safe_series_to_list(drv_df[col])

    result["cdc_credit_inputs"] = cdc_credit_inputs

    return result

df_to_json_custom

df_to_json_custom(drv_df: DataFrame, input_vars: List[str], inputs_key: str = 'inputs', metadata_cols: Optional[List[str]] = None, unwrap_single: bool = True) -> Dict[str, Any]

Convert a drv_df DataFrame to the partitioned JSON format, with a custom second-level key and automatic unwrapping.

Similar to the partitioned mode of df_to_json, but: - The key name of the second-level JSON can be customized (via inputs_key) - Arrays of length 1 in the second-level JSON are automatically unwrapped to scalars (via unwrap_single)

Parameters:

Name	Type	Description	Default
drv_df	DataFrame	Source DataFrame.	required
input_vars	List[str]	Names of the columns placed in the second-level JSON.	required
inputs_key	str	Key name of the second-level JSON; default "inputs".	'inputs'
metadata_cols	Optional[List[str]]	Top-level metadata columns. If None, they are derived automatically (the columns that are not in input_vars).	None
unwrap_single	bool	Whether to unwrap arrays of length 1 in the second-level JSON to scalars. Default True.	True

Returns:

Type	Description
Dict[str, Any]	{ "": , ..., "": { "": , ... } }

Examples:

>>> df = pd.DataFrame({'req': ['a','a'], 'x': [1,2], 'y': [3,4]})
>>> df_to_json_custom(df, input_vars=['x','y'], inputs_key='features')
{'req': 'a', 'features': {'x': [1,2], 'y': [3,4]}}

>>> df_single = pd.DataFrame({'req': ['a'], 'x': [1], 'y': [3]})
>>> df_to_json_custom(df_single, input_vars=['x','y'], inputs_key='features')
{'req': 'a', 'features': {'x': 1, 'y': 3}}

Source code in Modeling_Tool/Core/Json_Data_Converter.py

def df_to_json_custom(
    drv_df: pd.DataFrame,
    input_vars: List[str],
    inputs_key: str = "inputs",
    metadata_cols: Optional[List[str]] = None,
    unwrap_single: bool = True,
) -> Dict[str, Any]:
    """Convert a drv_df DataFrame to the partitioned JSON format, with a custom second-level key and automatic unwrapping.

    Similar to the partitioned mode of df_to_json, but:
      - The key name of the second-level JSON can be customized (via inputs_key)
      - Arrays of length 1 in the second-level JSON are automatically unwrapped to scalars (via unwrap_single)

    Parameters
    ----------
    drv_df : pd.DataFrame
        Source DataFrame.
    input_vars : List[str]
        Names of the columns placed in the second-level JSON.
    inputs_key : str
        Key name of the second-level JSON; default "inputs".
    metadata_cols : Optional[List[str]]
        Top-level metadata columns. If None, they are derived automatically (the columns that are not in input_vars).
    unwrap_single : bool
        Whether to unwrap arrays of length 1 in the second-level JSON to scalars. Default True.

    Returns
    -------
    Dict[str, Any]
        {
            "<meta>": <scalar>,
            ...,
            "<inputs_key>": {
                "<var>": <scalar_or_array>,
                ...
            }
        }

    Examples
    --------
    >>> df = pd.DataFrame({'req': ['a','a'], 'x': [1,2], 'y': [3,4]})
    >>> df_to_json_custom(df, input_vars=['x','y'], inputs_key='features')
    {'req': 'a', 'features': {'x': [1,2], 'y': [3,4]}}

    >>> df_single = pd.DataFrame({'req': ['a'], 'x': [1], 'y': [3]})
    >>> df_to_json_custom(df_single, input_vars=['x','y'], inputs_key='features')
    {'req': 'a', 'features': {'x': 1, 'y': 3}}
    """
    # ── Argument validation ──
    missing_cols = set(input_vars) - set(drv_df.columns)
    if missing_cols:
        raise KeyError(
            f"Columns in input_vars do not exist in the DataFrame: {missing_cols}"
        )

    if metadata_cols is None:
        metadata_cols = [c for c in drv_df.columns if c not in input_vars]

    missing_meta = set(metadata_cols) - set(drv_df.columns)
    if missing_meta:
        raise KeyError(
            f"Columns in metadata_cols do not exist in the DataFrame: {missing_meta}"
        )

    # Check that the metadata column values are consistent across the whole DataFrame
    for col in metadata_cols:
        unique_vals = drv_df[col].drop_duplicates()
        if len(unique_vals) > 1:
            raise ValueError(
                f"Metadata column '{col}' has multiple distinct values: {unique_vals.to_list()}. "
                f"Metadata columns are expected to be constant across all rows; check the data or adjust the metadata_cols argument."
            )

    # ── Build the output ──
    result: Dict[str, Any] = {}

    # 1) Metadata scalars
    first_row = drv_df.iloc[0]
    for col in metadata_cols:
        result[col] = _safe_json_value(first_row[col])

    # 2) Second-level JSON: custom key + automatic unwrapping of single elements
    inputs: Dict[str, Any] = {}
    for col in input_vars:
        arr = _safe_series_to_list(drv_df[col])
        if unwrap_single and len(arr) == 1:
            inputs[col] = arr[0]
        else:
            inputs[col] = arr

    result[inputs_key] = inputs

    return result

json_to_df

json_to_df(json_data: Union[str, Dict[str, Any]], input_vars: Optional[List[str]] = None, metadata_cols: Optional[List[str]] = None) -> DataFrame

Restore a DataFrame from the JSON format (the three formats are detected automatically).

Three modes: 1) Row-Oriented mode (the JSON has a cdc_query_credits key) - NEW Top-level scalars are metadata and cdc_query_credits is an array of objects, one row per object. Output: the metadata columns are broadcast to all rows + the object fields are expanded into columns.

   {
       "requestId": "req@123", "pullLogId": 3836171, ...,
       "cdc_query_credits": [
           {"_id": "id1", "montoPagar": 922, ...},
           {"_id": "id2", "montoPagar": 0,   ...}
       ]
   }

2) Partitioned mode (the JSON has a cdc_credit_inputs key) The array columns are taken from cdc_credit_inputs; all other top-level keys are metadata scalars.

   {
       "requestid": "req_001", "pulllogid": 3836171, ...,
       "cdc_credit_inputs": {
           "account_open_days": [2381, 4493],
           "pagoactual": ["V", "V"]
       }
   }

3) Flat mode (the JSON has neither cdc_credit_inputs nor cdc_query_credits) All top-level keys are column names; list values are expanded into rows and scalar values are broadcast.

Parameters:

Name	Type	Description	Default
json_data	Union[str, Dict[str, Any]]	JSON string or dictionary.	required
input_vars	Optional[List[str]]	Model input feature column names in partitioned mode. If None, the JSON format is detected automatically. Note: this argument is ignored in row-oriented mode (all fields in cdc_query_credits are expanded).	None
metadata_cols	Optional[List[str]]	Metadata column names. If None, they are derived automatically (top-level keys other than cdc_*).	None

Returns:

Type	Description
DataFrame	The restored DataFrame, with the metadata columns first and the data columns after.

Raises:

Type	Description
ValueError	If the array lengths are inconsistent, or cdc_query_credits is not an array.

Source code in Modeling_Tool/Core/Json_Data_Converter.py

def json_to_df(
    json_data: Union[str, Dict[str, Any]],
    input_vars: Optional[List[str]] = None,
    metadata_cols: Optional[List[str]] = None,
) -> pd.DataFrame:
    """Restore a DataFrame from the JSON format (the three formats are detected automatically).

    Three modes:
      1) Row-Oriented mode (the JSON has a cdc_query_credits key) - NEW
           Top-level scalars are metadata and cdc_query_credits is an array of objects, one row per object.
           Output: the metadata columns are broadcast to all rows + the object fields are expanded into columns.

           {
               "requestId": "req@123", "pullLogId": 3836171, ...,
               "cdc_query_credits": [
                   {"_id": "id1", "montoPagar": 922, ...},
                   {"_id": "id2", "montoPagar": 0,   ...}
               ]
           }

      2) Partitioned mode (the JSON has a cdc_credit_inputs key)
           The array columns are taken from cdc_credit_inputs; all other top-level keys are metadata scalars.

           {
               "requestid": "req_001", "pulllogid": 3836171, ...,
               "cdc_credit_inputs": {
                   "account_open_days": [2381, 4493],
                   "pagoactual": ["V", "V"]
               }
           }

      3) Flat mode (the JSON has neither cdc_credit_inputs nor cdc_query_credits)
           All top-level keys are column names; list values are expanded into rows and scalar values are broadcast.

    Parameters
    ----------
    json_data : Union[str, Dict[str, Any]]
        JSON string or dictionary.
    input_vars : Optional[List[str]]
        Model input feature column names in partitioned mode. If None, the JSON format is detected automatically.
        Note: this argument is ignored in row-oriented mode (all fields in cdc_query_credits are expanded).
    metadata_cols : Optional[List[str]]
        Metadata column names. If None, they are derived automatically (top-level keys other than cdc_*).

    Returns
    -------
    pd.DataFrame
        The restored DataFrame, with the metadata columns first and the data columns after.

    Raises
    ------
    ValueError
        If the array lengths are inconsistent, or cdc_query_credits is not an array.
    """
    if isinstance(json_data, str):
        data = json.loads(json_data)
    else:
        data = json_data

    has_cdc_inputs = "cdc_credit_inputs" in data
    has_cdc_credits = "cdc_query_credits" in data

    # ═══════════════════════════════════════════════════════════════════════
    # Mode 1: Row-Oriented — cdc_query_credits (NEW)
    # ═══════════════════════════════════════════════════════════════════════
    if has_cdc_credits:
        credits = data["cdc_query_credits"]
        if not isinstance(credits, list):
            raise ValueError(
                f"cdc_query_credits must be an array, got type {type(credits).__name__}"
            )

        # Build the DataFrame from the array of objects
        if len(credits) == 0:
            df = pd.DataFrame()
        else:
            df = pd.DataFrame(credits)

        # Broadcast the top-level metadata scalars to all rows
        for key, val in data.items():
            if key == "cdc_query_credits":
                continue
            if len(df) == 0:
                df[key] = pd.Series(dtype=type(val) if val is not None else object)
            else:
                df[key] = val

        # Column order: metadata first → data fields after
        meta_cols_result = [k for k in data if k != "cdc_query_credits"]
        credit_cols_result = [c for c in df.columns if c not in meta_cols_result]
        df = df[meta_cols_result + credit_cols_result]

        return df

    # ═══════════════════════════════════════════════════════════════════════
    # Modes 2 & 3: cdc_credit_inputs partitioned mode / flat mode
    # ═══════════════════════════════════════════════════════════════════════

    # ── Auto-detection: handling when input_vars is not specified ──
    if input_vars is None:
        if has_cdc_inputs:
            # Partitioned mode: input_vars = all keys of cdc_credit_inputs
            input_vars = list(data["cdc_credit_inputs"].keys())
        else:
            # Flat mode: every list value is a column
            input_vars = []  # no special input_vars distinction

    # ── Mode 3: flat mode: all top-level keys become columns directly ──
    if not has_cdc_inputs:
        # Find all list columns and scalar columns
        list_cols = {}
        scalar_cols = {}
        n_rows = 0
        for key, val in data.items():
            if isinstance(val, list):
                list_cols[key] = val
                if n_rows == 0:
                    n_rows = len(val)
                elif len(val) != n_rows:
                    raise ValueError(
                        f"Inconsistent array lengths: expected {n_rows}, but '{key}' has length {len(val)}"
                    )
            else:
                scalar_cols[key] = val

        if n_rows == 0:
            # With no list columns: if there are scalar columns, build a single-row DataFrame;
            # otherwise (empty JSON) return an empty DataFrame.
            if scalar_cols:
                return pd.DataFrame([scalar_cols])
            return pd.DataFrame()

        df = pd.DataFrame(list_cols)
        for key, val in scalar_cols.items():
            df[key] = val
        return df

    # ── Mode 2: partitioned mode (original logic) ──
    cdc_inputs = data["cdc_credit_inputs"]

    missing_inputs = set(input_vars) - set(cdc_inputs.keys())
    if missing_inputs:
        raise ValueError(
            f"Fields in input_vars do not exist in cdc_credit_inputs: {missing_inputs}"
        )

    # Check that all array lengths are consistent
    lengths: Dict[str, int] = {}
    for key in cdc_inputs:
        if isinstance(cdc_inputs[key], list):
            lengths[key] = len(cdc_inputs[key])

    if lengths:
        ref_key = next((k for k in input_vars if k in lengths), list(lengths.keys())[0])
        ref_len = lengths[ref_key]
        for key, length in lengths.items():
            if length != ref_len:
                raise ValueError(
                    f"Inconsistent array lengths in cdc_credit_inputs: "
                    f"'{ref_key}' has length {ref_len}, but '{key}' has length {length}"
                )
        n_rows = ref_len
    else:
        n_rows = 0

    # Build the DataFrame
    df = pd.DataFrame({col: cdc_inputs[col] for col in input_vars})

    extra_input_cols = [k for k in cdc_inputs if k not in input_vars]
    for col in extra_input_cols:
        df[col] = cdc_inputs[col]

    if metadata_cols is None:
        metadata_cols = [k for k in data if k != "cdc_credit_inputs"]

    for col in metadata_cols:
        if col in data:
            df[col] = data[col]

    ordered_cols = (
        [c for c in metadata_cols if c in df.columns]
        + [c for c in input_vars if c in df.columns and c not in metadata_cols]
        + [c for c in extra_input_cols if c in df.columns]
    )
    df = df[ordered_cols]

    return df

df_to_json_string

df_to_json_string(drv_df: DataFrame, input_vars: Optional[List[str]] = None, metadata_cols: Optional[List[str]] = None, indent: Optional[int] = 2, ensure_ascii: bool = False) -> str

Convenience wrapper around df_to_json that returns the JSON string directly.

Parameters:

Name	Type	Description	Default
drv_df	DataFrame	Source DataFrame, one record per row.	required
input_vars	Optional[List[str]]	List of model input feature column names; None selects the flat mode of df_to_json.	None
metadata_cols	Optional[List[str]]	Metadata columns to specify explicitly in partitioned mode; ignored in flat mode.	None
indent	Optional[int]	Number of spaces used to indent the JSON; None gives compact single-line output.	2
ensure_ascii	bool	Whether to escape non-ASCII characters as \uXXXX.	False

Returns:

Type	Description
str	The JSON text. NaN and infinite values are written as null and numpy scalars as plain numbers.

Raises:

Type	Description
ValueError	In partitioned mode, if the metadata column values are inconsistent.
KeyError	If a column in input_vars does not exist in the DataFrame.

Source code in Modeling_Tool/Core/Json_Data_Converter.py

def df_to_json_string(
    drv_df: pd.DataFrame,
    input_vars: Optional[List[str]] = None,
    metadata_cols: Optional[List[str]] = None,
    indent: Optional[int] = 2,
    ensure_ascii: bool = False,
) -> str:
    """Convenience wrapper around df_to_json that returns the JSON string directly.

    Parameters
    ----------
    drv_df : pd.DataFrame
        Source DataFrame, one record per row.
    input_vars : Optional[List[str]], default None
        List of model input feature column names; None selects the flat mode of ``df_to_json``.
    metadata_cols : Optional[List[str]], default None
        Metadata columns to specify explicitly in partitioned mode; ignored in flat mode.
    indent : Optional[int], default 2
        Number of spaces used to indent the JSON; None gives compact single-line output.
    ensure_ascii : bool, default False
        Whether to escape non-ASCII characters as \\uXXXX.

    Returns
    -------
    str
        The JSON text. NaN and infinite values are written as ``null`` and numpy scalars as plain numbers.

    Raises
    ------
    ValueError
        In partitioned mode, if the metadata column values are inconsistent.
    KeyError
        If a column in ``input_vars`` does not exist in the DataFrame.
    """
    result = df_to_json(drv_df, input_vars, metadata_cols)
    result = _sanitize_for_json(result)
    return json.dumps(result, indent=indent, ensure_ascii=ensure_ascii)

json_string_to_df

json_string_to_df(json_string: str, input_vars: Optional[List[str]] = None, metadata_cols: Optional[List[str]] = None) -> DataFrame

Convenience wrapper around json_to_df that accepts a JSON string as input.

Parameters:

Name	Type	Description	Default
json_string	str	JSON text in one of the formats that json_to_df recognizes.	required
input_vars	Optional[List[str]]	Model input feature column names; see json_to_df.	None
metadata_cols	Optional[List[str]]	Metadata column names; see json_to_df.	None

Returns:

Type	Description
DataFrame	The restored table.

Source code in Modeling_Tool/Core/Json_Data_Converter.py

def json_string_to_df(
    json_string: str,
    input_vars: Optional[List[str]] = None,
    metadata_cols: Optional[List[str]] = None,
) -> pd.DataFrame:
    """Convenience wrapper around json_to_df that accepts a JSON string as input.

    Parameters
    ----------
    json_string : str
        JSON text in one of the formats that ``json_to_df`` recognizes.
    input_vars : Optional[List[str]], default None
        Model input feature column names; see ``json_to_df``.
    metadata_cols : Optional[List[str]], default None
        Metadata column names; see ``json_to_df``.

    Returns
    -------
    pd.DataFrame
        The restored table.
    """
    return json_to_df(json_string, input_vars, metadata_cols)

df_to_json_file

df_to_json_file(drv_df: DataFrame, output_path: str, input_vars: Optional[List[str]] = None, metadata_cols: Optional[List[str]] = None, indent: Optional[int] = 2, ensure_ascii: bool = False) -> str

Convert a drv_df DataFrame to the expected JSON format and write it to a .json file.

Parameters:

Name	Type	Description	Default
drv_df	DataFrame	Source DataFrame, one credit bureau account record per row.	required
output_path	str	Path of the output .json file.	required
input_vars	Optional[List[str]]	List of model input feature column names. With None the flat mode of df_to_json is used and every column becomes an array.	None
metadata_cols	Optional[List[str]]	Metadata column names to specify explicitly (partitioned mode only). If None, they are derived automatically.	None
indent	Optional[int]	Number of spaces used to indent the JSON. None means compact output (single line).	2
ensure_ascii	bool	Whether to escape non-ASCII characters as \uXXXX. The default False keeps non-ASCII characters (such as CJK text) as they are.	False

Returns:

Type	Description
str	Absolute path of the written file.

Source code in Modeling_Tool/Core/Json_Data_Converter.py

def df_to_json_file(
    drv_df: pd.DataFrame,
    output_path: str,
    input_vars: Optional[List[str]] = None,
    metadata_cols: Optional[List[str]] = None,
    indent: Optional[int] = 2,
    ensure_ascii: bool = False,
) -> str:
    """Convert a drv_df DataFrame to the expected JSON format and write it to a .json file.

    Parameters
    ----------
    drv_df : pd.DataFrame
        Source DataFrame, one credit bureau account record per row.
    output_path : str
        Path of the output .json file.
    input_vars : Optional[List[str]], default None
        List of model input feature column names. With None the flat mode of ``df_to_json`` is used and every column
        becomes an array.
    metadata_cols : Optional[List[str]], default None
        Metadata column names to specify explicitly (partitioned mode only). If None, they are derived automatically.
    indent : Optional[int], default 2
        Number of spaces used to indent the JSON. None means compact output (single line).
    ensure_ascii : bool, default False
        Whether to escape non-ASCII characters as \\uXXXX. The default False keeps non-ASCII characters (such as CJK text) as they are.

    Returns
    -------
    str
        Absolute path of the written file.
    """
    result = df_to_json(drv_df, input_vars, metadata_cols)
    result = _sanitize_for_json(result)
    with open(output_path, 'w', encoding='utf-8') as f:
        json.dump(result, f, indent=indent, ensure_ascii=ensure_ascii)
    return os.path.abspath(output_path)

load_json_file

load_json_file(file_path: str) -> Dict[str, Any]

Load a .json file into a dictionary.

Parameters:

Name	Type	Description	Default
file_path	str	Path of the .json file.	required

Returns:

Type	Description
Dict[str, Any]	The parsed dictionary, whose structure follows the expected JSON format.

Source code in Modeling_Tool/Core/Json_Data_Converter.py

def load_json_file(file_path: str) -> Dict[str, Any]:
    """Load a .json file into a dictionary.

    Parameters
    ----------
    file_path : str
        Path of the .json file.

    Returns
    -------
    Dict[str, Any]
        The parsed dictionary, whose structure follows the expected JSON format.
    """
    with open(file_path, 'r', encoding='utf-8') as f:
        return json.load(f)

json_to_file

json_to_file(data: Dict[str, Any], output_path: str, indent: Optional[int] = 2, ensure_ascii: bool = False) -> str

Write a Python dict to a .json file.

Parameters:

Name	Type	Description	Default
data	Dict[str, Any]	Dictionary to write.	required
output_path	str	Path of the output .json file.	required
indent	Optional[int]	Number of spaces used to indent the JSON. None means compact single-line output; default 2.	2
ensure_ascii	bool	Whether to escape non-ASCII characters. Default False.	False

Returns:

Type	Description
str	Absolute path of the written file.

Source code in Modeling_Tool/Core/Json_Data_Converter.py

def json_to_file(
    data: Dict[str, Any],
    output_path: str,
    indent: Optional[int] = 2,
    ensure_ascii: bool = False,
) -> str:
    """Write a Python dict to a .json file.

    Parameters
    ----------
    data : Dict[str, Any]
        Dictionary to write.
    output_path : str
        Path of the output .json file.
    indent : Optional[int]
        Number of spaces used to indent the JSON. None means compact single-line output; default 2.
    ensure_ascii : bool
        Whether to escape non-ASCII characters. Default False.

    Returns
    -------
    str
        Absolute path of the written file.
    """
    with open(output_path, 'w', encoding='utf-8') as f:
        json.dump(_sanitize_for_json(data), f, indent=indent, ensure_ascii=ensure_ascii)
    return os.path.abspath(output_path)