Regresion con Scikit Learn

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

print(pd.__version__)

columnas = ['symboling', 'normalized-losses', 'make',
            'fuel-type', 'aspiration', 'num-of-doors',
            'body-style', 'drive-wheels', 'engine-location',
            'wheel-base', 'length', 'width', 'height',
            'curb-weight', 'engine-type', 'num-of-cylinders',
            'engine-size', 'fuel-system', 'bore', 'stroke',
            'compression-ratio', 'horsepower', 'peak-rpm',
            'city-mpg', 'highway-mpg', 'price']

url_data = 'https://raw.githubusercontent.com/jbrownlee/Datasets/master/auto_imports.csv'
auto_df = pd.read_csv(url_data,
                            header=None,
                            names=columnas,
                            na_values='?')
# Ver 5 registros aleatorios
auto_df.sample(5)

#Tamaño del dataset
auto_df.shape 

auto_df.info()

auto_df.isna().sum()

cols_categoricas = ["make", "fuel-type", "aspiration", "num-of-doors",
                    "body-style", "drive-wheels", "engine-location",
                    "engine-type", "num-of-cylinders", "fuel-system"]

auto_df[cols_categoricas] = auto_df[cols_categoricas].astype("category")

auto_df["num-of-doors"] = pd.Categorical(auto_df["num-of-doors"],
                                        categories=["two","four"],
                                        ordered=True)

auto_df["num-of-cylinders"] = pd.Categorical(auto_df["num-of-cylinders"],
                                      categories=["two", "three", "four",
                                                  "five", "six", "eight",
                                                  "twelve"],
                                      ordered=True)

auto_df.info()

auto_df.describe()

auto_df.head()

auto_df.to_parquet('automobile_processed.parquet',
                      index=False)

cols_categoricas

cols_cate_problemas = ["make", "fuel-system", "num-of-cylinders"]
for col in cols_cate_problemas:
    print(auto_df[col].value_counts())
    print("")

cond_1 = "`fuel-system` in ['mfi', 'spfi']"
cond_2 = "`num-of-cylinders` in ['three', 'twelve']"
cond_3 = "make == 'mercury'"
auto_df.query(f"{cond_1} | {cond_2} | {cond_3}")

#listado de variables numericas excepto price
cols_numericas = (auto_df
                  .drop(columns=["price"])
                  .select_dtypes(include=np.number)
                  .columns.tolist())
cols_numericas

len(cols_numericas)

# son 15 variables numericas 
# el plot seran 5 filas y 3 columnas

fig, axes = plt.subplots(5, 3, figsize=(15, 15))
axes = axes.flatten()
for i, col in enumerate(cols_numericas):
    sns.regplot(data=auto_df,
                x=col, y="price",
                ax=axes[i])
    axes[i].set_title(col)
plt.tight_layout()
plt.show()

# mover la columna price al inicio en el dataframe auto_df
# para que los valores de correlacion se vean al inicio

cols = auto_df.columns.tolist()
cols.insert(0, cols.pop(cols.index("price")))
auto_df = auto_df.reindex(columns= cols)

automobile_corr = auto_df.corr(numeric_only=True)

fig, ax = plt.subplots(figsize=(12, 10))

sns.heatmap(automobile_corr, annot=True, fmt=".2f");

len(cols_categoricas)

# son 15 variables numericas 
# el plot seran 5 filas y 3 columnas

fig, axes = plt.subplots(5, 2, figsize=(15, 15))
axes = axes.flatten()
for i, col in enumerate(cols_categoricas):
    sns.boxplot(data=auto_df,
                x="price", y=col,
                ax=axes[i])
    axes[i].set_title(col)
plt.tight_layout()
plt.show()

# librerias para el preprocesamiento de datos
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer

cols_numericas = ['symboling','normalized-losses',
                  'wheel-base','length', 'width',
                  'height', 'curb-weight', 
                  'engine-size', 'bore', 'stroke', 
                  'compression-ratio', 'horsepower',
                  'peak-rpm','city-mpg',
                  'highway-mpg']

cols_categoricas = ["make", "fuel-type", "aspiration",
                    "body-style", "drive-wheels",
                    "engine-location", "engine-type",
                    "fuel-system"]

cols_categoricas_ord = [ "num-of-doors", "num-of-cylinders"]

numeric_pipe = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='median'))])

categorical_pipe = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='most_frequent')),
    ('onehot', OneHotEncoder(handle_unknown='ignore'))])

categorical_ord_pipe = Pipeline(steps=[
    ('ordenc', OrdinalEncoder(handle_unknown='use_encoded_value',
                              unknown_value=np.nan)),
    ('imputer', SimpleImputer(strategy='most_frequent'))])

preprocessor = ColumnTransformer(
    transformers=[
        ('numericas', numeric_pipe, cols_numericas),
        ('categoricas', categorical_pipe, cols_categoricas),
        ('categoricas ordinales', categorical_ord_pipe, cols_categoricas_ord)
        ])

preprocessor

from sklearn.model_selection import train_test_split

X_features = auto_df.drop('price', axis='columns')
Y_target = auto_df['price']

x_train, x_test, y_train, y_test = train_test_split(X_features,
                                                    Y_target,
                                                    test_size=0.2,
                                                    random_state=42)

x_train.shape, y_train.shape

x_test.shape, y_test.shape

from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error


from sklearn.dummy import DummyRegressor
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.svm import SVR
from sklearn.neighbors import KNeighborsRegressor
from sklearn.tree import DecisionTreeRegressor

import warnings
warnings.filterwarnings("ignore")

result_dict = {}

FEATURES = list(x_train.columns)
FEATURES

def entrenar_modelo(modelo,
                    preprocessor: ColumnTransformer,                
                    x_data: pd.DataFrame, 
                    y_data: pd.Series,                    
                    test_frac:float=0.2,
                  ):
    
    """entrenar_modelo
    
    Funcion para entrenar y evaluar un modelo
    
    Args:
      modelo (scikit-learn model ): modelo de ML
      preprocessor (ColumnTransformer): preprocesador de datos
      x_data (pd.DataFrame): datos de entrada
      y_data (pd.Series): datos de salida
      test_frac (float): fraccion de datos para el conjunto de prueba
      
    Returns:
      dict: diccionario con los puntajes de entrenamiento y prueba        
    """

    # dividir el dataset en entrenamiento y prueba
    x_train, x_test, y_train, y_test = train_test_split(x_data, y_data,
                                                        random_state=42,
                                                        test_size=test_frac)
    # crear el pipeline con el preprocesador y el modelo
    regressor_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                                     ("model", modelo)])
    
    # entrenar el pipeline de regresion
    model = regressor_pipe.fit(x_train, y_train)
    
    y_pred_train = model.predict(x_train)
    
    # predecir con el pipeline de regresion
    y_pred = model.predict(x_test)
    
    train_score = mean_absolute_error(y_train, y_pred_train)
    test_score = mean_absolute_error(y_test, y_pred)
    print("Entrenamiento_score : " , train_score)
    print("Prueba_score : ", test_score)

    return {
            'Entrenamiento_score': train_score,
            'Prueba_score': test_score
           }     

# Funcion para comparar los resultados de los modelos
#se van a almacenar en un diccionario
def compare_results():
    for key in result_dict:
        print('Regresion: ', key)
        print('Entrenamiento score', result_dict[key]['Entrenamiento_score'])
        print('Prueba score', result_dict[key]['Prueba_score'])
        print()

result_dict['Dummy Regressor'] = entrenar_modelo(DummyRegressor(strategy='median'),
                                                 preprocessor,
                                                 x_train,
                                                 y_train)

result_dict['Linear Regressor'] = entrenar_modelo(LinearRegression(),
                                                 preprocessor,
                                                 x_train,
                                                 y_train)

result_dict['Lasso'] = entrenar_modelo(Lasso(alpha=0.5),
                                       preprocessor,
                                       x_train,
                                       y_train)

compare_results()

result_dict['Ridge'] = entrenar_modelo(Ridge(alpha=0.5),
                                       preprocessor,
                                       x_train,
                                       y_train)

result_dict['Elasticnet'] = entrenar_modelo(ElasticNet(alpha=1, l1_ratio=0.5,
                                                       max_iter= 100000, 
                                                       warm_start= True),
                                            preprocessor,
                                            x_train,
                                            y_train)

result_dict['SVR'] = entrenar_modelo(SVR(kernel='linear',
                                         epsilon=0.05,
                                         C=0.3),
                                     preprocessor,
                                     x_train,
                                     y_train)

result_dict['KNR'] = entrenar_modelo(KNeighborsRegressor(n_neighbors=10),
                                     preprocessor,
                                     x_train,
                                     y_train)

result_dict['Decision Tree'] = entrenar_modelo(DecisionTreeRegressor(max_depth=2),
                                               preprocessor,
                                               x_train,
                                               y_train)

compare_results()

# Crear un diccionario solo con los resultados de prueba de cada modelo
nombre_modelos = result_dict.keys()
resultados_train = {} # crear diccionario vacio
resultados_test = {} # crear diccionario vacio
for nombre in nombre_modelos:
    resultados_train[nombre] = result_dict[nombre]['Entrenamiento_score']
    resultados_test[nombre] = result_dict[nombre]['Prueba_score']

df_comparacion = pd.DataFrame([resultados_train, resultados_test],
                              index=['train', 'test'])    

# Plot the bar chart
fig, ax = plt.subplots(figsize=(10, 4))
df_comparacion.T.plot(kind='bar', ax=ax)

# Adjust the layout
ax.set_ylabel('MAE score')
ax.set_title('Comparacion de Modelos [MAE] ')

# Set the x-tick labels inside the bars and rotate by 90 degrees
ax.set_xticks(range(len(df_comparacion.columns)))
ax.set_xticklabels([])

# Draw the x-tick labels inside the bars rotated by 90 degrees
for i, label in enumerate(df_comparacion.columns):
    bar_center = (df_comparacion.loc['train', label] +
                  df_comparacion.loc['test', label]) / 2
    ax.text(i, bar_center, label, ha='center',
            va='center_baseline', rotation=45)

#plot line at DUmmerRegressor result
ax.axhline(df_comparacion['Dummy Regressor']['test'],
           color='red',
           linestyle='--',
           alpha=0.8)

plt.tight_layout()

# lista para almacenar cada uno los modelos seleccionados para el cross validation
models = []

# Alamcenando los modelos como una tupla (nombre, modelo)
models.append(('LinearRegression',LinearRegression()))
models.append(('Lasso',Lasso(alpha=0.5)))
models.append(('Elastic_net',ElasticNet(alpha=1,
                                        l1_ratio=0.5, 
                                        max_iter= 100000, 
                                        warm_start= True)))
models.append(('Kneighbors',KNeighborsRegressor(n_neighbors=10)))
models.append(('Decision_tree',DecisionTreeRegressor(max_depth=2)))
models.append(('SVR',SVR(kernel='linear', epsilon=0.05, C=0.3)))

# Grabar los resultados de cada modelo
from sklearn import model_selection

#Semilla para obtener los mismos resultados de pruebas
seed = 2
results = []
names = []
scoring = 'neg_mean_absolute_error'
for name, model in models:
    # Kfold cross validation for model selection
    kfold = model_selection.KFold(n_splits=10)
    model_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                                 ("model", model)])
    #X train , y train
    cv_results = model_selection.cross_val_score(model_pipe, x_train, y_train,
                                                 cv=kfold, scoring=scoring)
    # la metrica neg_mean_absolute_error se debe convertir en positiva
    cv_results = np.abs(cv_results)
    
    results.append(cv_results)
    names.append(name)
    msg = f"({name}, {cv_results.mean()}, {cv_results.std()}"
    print(msg)

plt.figure(figsize = (8,4)) 
result_df = pd.DataFrame(results, index=names).T
sns.boxplot(data=result_df)
plt.title("Resultados de Cross Validation")
plt.show()

plt.figure(figsize = (8,4)) 
sns.lineplot(data=result_df)
plt.title("Resultados de cada Kfold")
plt.show()

from scipy.stats import f_oneway

model1 = result_df['LinearRegression']
model2 = result_df['Lasso']
model3 = result_df['Elastic_net']
model4 = result_df['Kneighbors']
model5 = result_df['Decision_tree']
model6 = result_df['SVR']

statistic, p_value = f_oneway(model1, model2, model3, model4, model5, model6)

print(f'Statistic: {statistic}')
print(f'p_value: {p_value}')

alpha = 0.05  # nivel de significancia

if p_value < alpha:
    print("Existe una diferencia estadísticamente "
          "significativa en los resultados de"
          " cross-validation de los modelos.")
else:
    print("No Existe una diferencia estadísticamente " 
          "significativa en los resultados de " 
          "cross-validation de los modelos.")

from sklearn.model_selection import GridSearchCV

parameters = {'model__alpha': [0.2, 0.4, 0.6, 0.7, 0.8, 0.9, 1.0]}

lasso_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                             ("model", Lasso())])

grid_search = GridSearchCV(lasso_pipe, parameters, cv=5,
                           return_train_score=True,
                           scoring='neg_mean_absolute_error')
grid_search.fit(x_train, y_train);

# la medida neg_mean_absolute_error se debe convertir en positiva
print(f"Mejor resultado = {abs(grid_search.best_score_)}")
print(f"Mejor std = {grid_search.cv_results_['std_test_score'][grid_search.best_index_]}")
print(f"Mejor parametros = {grid_search.best_params_}")

# Para ver todos los resultados del cross validation
# No es necesario, solo es informativo para ver como varia el modelo
for i in range(len(parameters['model__alpha'])):
    print('Parametros: ', grid_search.cv_results_['params'][i])
    print('Promedio Score Prueba: ', np.abs(grid_search
                                            .cv_results_['mean_test_score'][i]))
    print('Std Score Prueba: ', grid_search.cv_results_['std_test_score'][i])
    
    print('Rank: ', grid_search.cv_results_['rank_test_score'][i])

grid_search.best_params_

lasso_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                             ("model", Lasso(alpha=grid_search
                                             .best_params_['model__alpha']))])

lasso_model = lasso_pipe.fit(x_train, y_train)

y_pred = lasso_model.predict(x_test)
y_pred_train = lasso_model.predict(x_train)

print('Entrenamiento score: ', mean_absolute_error(y_train, y_pred_train))
print('Prueba score: ', mean_absolute_error(y_test, y_pred))

parameters = {'model__n_neighbors': [10, 12, 14, 18, 20, 25, 30, 35, 50]}

knn_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                           ("model", KNeighborsRegressor())])

grid_search = GridSearchCV(knn_pipe, parameters, cv=5,
                           return_train_score=True,
                           scoring='neg_mean_absolute_error')
grid_search.fit(x_train, y_train);

print(f"Mejor resultado = {abs(grid_search.best_score_)}")
print(f"Mejor std = {grid_search.cv_results_['std_test_score'][grid_search.best_index_]}")
print(f"Mejor parametros = {grid_search.best_params_}")

knn_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                           ("model", KNeighborsRegressor(n_neighbors=grid_search.
                                                         best_params_['model__n_neighbors']))])

kneighbors_model = knn_pipe.fit(x_train, y_train)

y_pred = kneighbors_model.predict(x_test)
y_pred_train = kneighbors_model.predict(x_train)

print('Entrenamiento score: ', mean_absolute_error(y_train, y_pred_train))
print('Prueba score: ', mean_absolute_error(y_test, y_pred))

parameters = {'model__max_depth': [4, 5, 7, 9, 10],
              'model__max_features': [5, 10, 15, 20,30],
              'model__criterion': ['squared_error',
                                   'absolute_error',
                                   'poisson',
                                   'friedman_mse'],
              }

dt_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                           ("model", DecisionTreeRegressor())])

grid_search = GridSearchCV(dt_pipe, parameters, cv=5,
                           scoring='neg_mean_absolute_error',
                           return_train_score=True)
grid_search.fit(x_train, y_train);

print(f"Mejor resultado = {abs(grid_search.best_score_)}")
print(f"Mejor std = {grid_search.cv_results_['std_test_score'][grid_search.best_index_]}")
print(f"Mejor parametros = {grid_search.best_params_}")

model = DecisionTreeRegressor(max_depth=grid_search.best_params_['model__max_depth'],
                              criterion=grid_search.best_params_['model__criterion'])

dt_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                           ("model",model)])
decision_tree_model = dt_pipe.fit(x_train, y_train)

y_pred = decision_tree_model.predict(x_test)
y_pred_train = decision_tree_model.predict(x_train)

print('Entrenamiento score: ', mean_absolute_error(y_train, y_pred_train))
print('Prueba score: ', mean_absolute_error(y_test, y_pred))

parameters = {'model__epsilon': [0.05, 0.1, 0.2, 0.3],
              'model__C': [0.2, 0.3]}

svr_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                           ("model", SVR(kernel='linear'))])

grid_search = GridSearchCV(svr_pipe, parameters, cv=5,
                           scoring='neg_mean_absolute_error',
                           return_train_score=True)
grid_search.fit(x_train, y_train);

print(f"Mejor resultado = {abs(grid_search.best_score_)}")
print(f"Mejor std = {grid_search.cv_results_['std_test_score'][grid_search.best_index_]}")
print(f"Mejor parametros = {grid_search.best_params_}")

model = SVR(kernel='linear',
            epsilon=grid_search.best_params_['model__epsilon'], 
            C=grid_search.best_params_['model__C'])
svr_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                           ("model", model)]) 
svr_model = svr_pipe.fit(x_train, y_train)

y_pred = svr_model.predict(x_test)
y_pred_train = svr_model.predict(x_train)

print('Entrenamiento score: ', mean_absolute_error(y_train, y_pred_train))
print('Prueba score: ', mean_absolute_error(y_test, y_pred))

from sklearn.metrics import PredictionErrorDisplay
PredictionErrorDisplay.from_predictions(y_true=y_test,
                                        y_pred=y_pred,
                                        kind="actual_vs_predicted");

PredictionErrorDisplay.from_predictions(y_true=y_test,
                                        y_pred=y_pred,
                                        kind="residual_vs_predicted");

from sklearn.inspection import permutation_importance

imps = permutation_importance(svr_pipe, x_test, y_test,
                              n_repeats = 5,
                              scoring= "neg_mean_absolute_error",
                              n_jobs=-1, random_state=42)

fig= plt.figure(figsize=(10,8))
perm_sorted_idx = imps.importances_mean.argsort()
plt.boxplot(imps.importances[perm_sorted_idx].T, vert=False,
            labels=x_test.columns[perm_sorted_idx])
plt.title("Permutation Importances (test set)");

#los valores estan en orden ascendente, entonces se deben tomar los ultimos 8
# se puede comprobar al comparar con el grafico de barras
cols_seleccionadas = x_test.columns[perm_sorted_idx][-8:].tolist()
cols_seleccionadas

x_train = x_train[cols_seleccionadas]
x_test = x_test[cols_seleccionadas]
x_train.head()

preprocessor = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='median'))])

model = SVR(kernel='linear',
            epsilon=0.05, 
            C=0.3)

svr_pipe = Pipeline(steps=[("preprocessor", preprocessor),
                           ("model", model)]) 
svr_model = svr_pipe.fit(x_train, y_train)

y_pred = svr_model.predict(x_test)
y_pred_train = svr_model.predict(x_train)

print('Entrenamiento score: ', mean_absolute_error(y_train, y_pred_train))
print('Prueba score: ', mean_absolute_error(y_test, y_pred))

columnas = ['width', 'compression-ratio', 'length',
            'wheel-base', 'peak-rpm', 'horsepower',
            'engine-size', 'price']
url_data = 'https://raw.githubusercontent.com/jbrownlee/Datasets/master/auto_imports.csv'
auto_df = pd.read_csv(url_data,
                      header=None,
                      names=columnas,
                      na_values='?')

X = auto_df.drop('price', axis='columns')
y = auto_df['price']

svr_model = svr_pipe.fit(X, y)

from joblib import dump# libreria de serializacion

# grabar el modelo en un archivo
dump(svr_model, 'svr_model-auto.joblib')

import pandas as pd
from joblib import load 

modelo = load('svr_model-auto.joblib')
modelo

# tomar dos datos de entrada para realizar la prediccion, eliminando la columna precio
datos_prueba = X. sample(2)
datos_prueba

# resultados de predicion con el modelo
modelo.predict(datos_prueba)