Kaggle入门赛之Titanic

2018-08-11

数据分析
1. 填充缺失值
Scale以及模型选择
参考

本文是初次参加Kaggle入门赛Titanic生还预测的一个过程记录，通过这个比赛，主要熟悉了pandas以及sklearn包的使用，对于常见的分类问题有了一定的了解。现将数据分析的过程记录如下。


import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import xgboost
from sklearn.preprocessing import OneHotEncoder, LabelEncoder,StandardScaler,MinMaxScaler
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier,RandomForestRegressor
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve
from mlxtend.classifier import StackingClassifier

数据分析

train_df=pd.read_csv('data/train.csv')
test_df=pd.read_csv('data/test.csv')
data_all=pd.concat([train_df,test_df],ignore_index=True)
train_len=train_df.shape[0]
print(train_df.shape)
print(test_df.shape)

(891, 12)
(418, 11)

1	data_all.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1309 entries, 0 to 1308
Data columns (total 12 columns):
Age            1046 non-null float64
Cabin          295 non-null object
Embarked       1307 non-null object
Fare           1308 non-null float64
Name           1309 non-null object
Parch          1309 non-null int64
PassengerId    1309 non-null int64
Pclass         1309 non-null int64
Sex            1309 non-null object
SibSp          1309 non-null int64
Survived       891 non-null float64
Ticket         1309 non-null object
dtypes: float64(3), int64(4), object(5)
memory usage: 122.8+ KB

n = data_all.drop(['PassengerId','Survived'], axis=1).loc[:, data_all.isnull().any()].isnull().sum()
print('ALL:', data_all.shape[0])
print('-' * 30)
print(n.sort_values(ascending=False))

ALL: 1309
------------------------------
Cabin       1014
Age          263
Embarked       2
Fare           1
dtype: int64

填充缺失值

## Cabin缺失较多 考虑将其分为 有值和无值两类 或者直接弃用
# 这里直接弃用
## Fare只缺1个值，用均值或中值填充，这里采用中值填充
data_all['Fare'].fillna(data_all['Fare'].median(),inplace=True)
## Embarked缺失2个值，为object类型，这里考虑添加一组新值，以"None"填充
data_all['Embarked'].fillna('None',inplace=True)
## Age对于预测很重要，这里考虑用机器学习的方法对其进行预测填充，考虑到Age和SibSp,Parch以及Pclass，Fare等因素有关
# 分析Age的影响因素
# 性别可能对age有影响，先将sex转化为数值类型
# convert Sex into categorical value 0 for male and 1 for female
data_all["Sex"] = data_all["Sex"].map({"male": 0, "female":1})
g = sns.heatmap(data_all[["Age","Sex","SibSp","Parch","Pclass","Fare"]].corr(),cmap="BrBG",annot=True)
# 分析出Age与Sex关系较小，而与其它几个属性都有一定关系
# 考虑使用RF模型结合其它几个属性对Age进行预测填充
# 把已有的数值型特征取出来丢进Random Forest Regressor中
age_df = data_all[['Age', 'Fare', 'Parch', 'SibSp', 'Pclass']]
# 乘客分成已知年龄和未知年龄两部分
known_age = age_df[age_df.Age.notnull()].as_matrix()
unknown_age = age_df[age_df.Age.isnull()].as_matrix()
# y即目标年龄
y = known_age[:, 0]
# X即特征属性值
X = known_age[:, 1:]
# fit到RandomForestRegressor之中
rfr = RandomForestRegressor(random_state=0, n_estimators=2000, n_jobs=-1)
rfr.fit(X, y)
# 用得到的模型进行未知年龄结果预测
predictedAges = rfr.predict(unknown_age[:, 1::])
# 用得到的预测结果填补原缺失数据
data_all.loc[(data_all.Age.isnull()), 'Age'] = predictedAges

png

# 显示数值类型的统计结果
print('number describe:\n',data_all.drop(['PassengerId','Survived'],axis=1).describe(include=['number']).loc[['min', 'max', 'mean','std','count']].T.sort_values('max'))
# 看出Age的缺失值已经被填充

number describe:
          min       max       mean        std   count
Sex     0.00    1.0000   0.355997   0.478997  1309.0
Pclass  1.00    3.0000   2.294882   0.837836  1309.0
SibSp   0.00    8.0000   0.498854   1.041658  1309.0
Parch   0.00    9.0000   0.385027   0.865560  1309.0
Age     0.17   80.0000  29.876751  13.447012  1309.0
Fare    0.00  512.3292  33.281086  51.741500  1309.0

# 显示object类型的取值
print('object describe:\n')
n = data_all.select_dtypes(include=object)
for c in n.columns:
    print('{:<14}'.format(c),':',len(data_all[c].unique()),'\n',data_all[c].unique())

object describe:

Cabin          : 187 
 [nan 'C85' 'C123' 'E46' 'G6' 'C103' 'D56' 'A6' 'C23 C25 C27' 'B78' 'D33'
 'B30' 'C52' 'B28' 'C83' 'F33' 'F G73' 'E31' 'A5' 'D10 D12' 'D26' 'C110'
 'B58 B60' 'E101' 'F E69' 'D47' 'B86' 'F2' 'C2' 'E33' 'B19' 'A7' 'C49'
 'F4' 'A32' 'B4' 'B80' 'A31' 'D36' 'D15' 'C93' 'C78' 'D35' 'C87' 'B77'
 'E67' 'B94' 'C125' 'C99' 'C118' 'D7' 'A19' 'B49' 'D' 'C22 C26' 'C106'
 'C65' 'E36' 'C54' 'B57 B59 B63 B66' 'C7' 'E34' 'C32' 'B18' 'C124' 'C91'
 'E40' 'T' 'C128' 'D37' 'B35' 'E50' 'C82' 'B96 B98' 'E10' 'E44' 'A34'
 'C104' 'C111' 'C92' 'E38' 'D21' 'E12' 'E63' 'A14' 'B37' 'C30' 'D20' 'B79'
 'E25' 'D46' 'B73' 'C95' 'B38' 'B39' 'B22' 'C86' 'C70' 'A16' 'C101' 'C68'
 'A10' 'E68' 'B41' 'A20' 'D19' 'D50' 'D9' 'A23' 'B50' 'A26' 'D48' 'E58'
 'C126' 'B71' 'B51 B53 B55' 'D49' 'B5' 'B20' 'F G63' 'C62 C64' 'E24' 'C90'
 'C45' 'E8' 'B101' 'D45' 'C46' 'D30' 'E121' 'D11' 'E77' 'F38' 'B3' 'D6'
 'B82 B84' 'D17' 'A36' 'B102' 'B69' 'E49' 'C47' 'D28' 'E17' 'A24' 'C50'
 'B42' 'C148' 'B45' 'B36' 'A21' 'D34' 'A9' 'C31' 'B61' 'C53' 'D43' 'C130'
 'C132' 'C55 C57' 'C116' 'F' 'A29' 'C6' 'C28' 'C51' 'C97' 'D22' 'B10'
 'E45' 'E52' 'A11' 'B11' 'C80' 'C89' 'F E46' 'B26' 'F E57' 'A18' 'E60'
 'E39 E41' 'B52 B54 B56' 'C39' 'B24' 'D40' 'D38' 'C105']
Embarked       : 4 
 ['S' 'C' 'Q' 'None']
Name           : 1307 
 ['Braund, Mr. Owen Harris'
 'Cumings, Mrs. John Bradley (Florence Briggs Thayer)'
 'Heikkinen, Miss. Laina' ... 'Saether, Mr. Simon Sivertsen'
 'Ware, Mr. Frederick' 'Peter, Master. Michael J']
Ticket         : 929 
 ['A/5 21171' 'PC 17599' 'STON/O2. 3101282' '113803' '373450' '330877'
 '17463' '349909' '347742' '237736' 'PP 9549' '113783' 'A/5. 2151'
    ...
 'A.5. 3236' 'SOTON/O.Q. 3101262' '359309']

#对object特征分组
#处理Name属性
data_all['Title'] = data_all['Name']
# Cleaning name and extracting Title
for name_string in data_all['Name']:
    data_all['Title'] = data_all['Name'].str.extract('([A-Za-z]+)\.', expand=True)
# Replacing rare titles with more common ones
mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
               'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
data_all.replace({'Title': mapping}, inplace=True)
g = sns.factorplot(x="Title", y="Survived",  data=data_all.iloc[:train_len,:],
                   size=6, kind="bar", palette="muted")
g.despine(left=True)
g = g.set_ylabels("survival probability")
# 分析得出女人小孩确实生存率更高

png

1 2	# 对Title进行编码 data_all['Title'].replace(['Mr','Miss','Mrs','Master','Rev','Dr'],[0,1,2,3,4,4],inplace=True)

# 家族属性特征提取
# 合并Parch属性和SibSp属性，添加Family_size属性
data_all['Family_size']=data_all['Parch']+data_all['SibSp']+1
g = sns.factorplot(x="Parch", y="Survived",  data=data_all.iloc[:train_len,:],
                   size=6, kind="bar", palette="muted")
g.despine(left=True)
g = g.set_ylabels("survival probability")
g = sns.factorplot(x="SibSp", y="Survived",  data=data_all.iloc[:train_len,:],
                   size=6, kind="bar", palette="muted")
g.despine(left=True)
g = g.set_ylabels("survival probability")
g = sns.factorplot(x="Family_size", y="Survived",  data=data_all.iloc[:train_len,:],
                   size=6, kind="bar", palette="muted")
g.despine(left=True)
g = g.set_ylabels("survival probability")
g = sns.countplot(x="Family_size", hue="Survived", data=data_all.iloc[:train_len,:])

png

# 家族属性深挖
data_all['Last_Name'] = data_all['Name'].apply(lambda x: str.split(x, ",")[0])
DEFAULT_SURVIVAL_VALUE = 0.5
data_all['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in data_all[['Survived', 'Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
                                'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
    if (len(grp_df) != 1):
        # A Family group is found.
        for ind, row in grp_df.iterrows():
            smax = grp_df.drop(ind)['Survived'].max()
            smin = grp_df.drop(ind)['Survived'].min()
            passID = row['PassengerId']
            if (smax == 1.0):
                data_all.loc[data_all['PassengerId'] == passID, 'Family_Survival'] = 1
            elif (smin == 0.0):
                data_all.loc[data_all['PassengerId'] == passID, 'Family_Survival'] = 0
for _, grp_df in data_all.groupby('Ticket'):
    if (len(grp_df) != 1):
        for ind, row in grp_df.iterrows():
            if (row['Family_Survival'] == 0) | (row['Family_Survival'] == 0.5):
                smax = grp_df.drop(ind)['Survived'].max()
                smin = grp_df.drop(ind)['Survived'].min()
                passID = row['PassengerId']
                if (smax == 1.0):
                    data_all.loc[data_all['PassengerId'] == passID, 'Family_Survival'] = 1
                elif (smin == 0.0):
                    data_all.loc[data_all['PassengerId'] == passID, 'Family_Survival'] = 0
# 查看Family_Survival分布
g = sns.countplot(x="Family_Survival", hue="Survived", data=data_all.iloc[:train_len,:])

png

# 将连续数值分成bins
# 处理Fare
# 显示Fare的分布 
g = sns.distplot(data_all["Fare"], color="m", label="Skewness : %.2f"%(data_all["Fare"].skew()))
g = g.legend(loc="best")

png

从上面可以看出Fare的分布比较不均匀，对其通过log尺度变换进行处理。

# Apply log to Fare to reduce skewness distribution
data_all["Fare"] = data_all["Fare"].map(lambda i: np.log(i) if i > 0 else 0)
g = sns.distplot(data_all["Fare"], color="b", label="Skewness : %.2f"%(data_all["Fare"].skew()))
g = g.legend(loc="best")

png

# 现在对Fare分成bins
# 这里选择qcut方法，对频率进行等距分箱
n_bins=5
data_all['FareBin'] = pd.qcut(data_all['Fare'], n_bins)
label = LabelEncoder()
data_all['FareBin_Code'] = label.fit_transform(data_all['FareBin'])
g = sns.countplot(x="FareBin_Code",hue='Survived',data=data_all.iloc[:train_len,:])

png

1
2
3

# 同样对年龄进行处理
g = sns.distplot(data_all["Age"], color="m", label="Skewness : %.2f"%(data_all["Age"].skew()))
g = g.legend(loc="best")

png

# 年龄的分布就比较均匀，不需要做任何预处理
# 这里用cut方法，对年龄选择等距分箱，分成n_bins_age个年龄段
n_bins_age=8
data_all['AgeBin'] = pd.cut(data_all['Age'], n_bins_age)
label = LabelEncoder()
data_all['AgeBin_Code'] = label.fit_transform(data_all['AgeBin'])
g = sns.countplot(x="AgeBin",hue='Survived',data=data_all.iloc[:train_len,:])

png

1 2	# 处理Embarked属性 g = sns.countplot(x="Embarked",data=data_all)

png

1	g = sns.countplot(x="Embarked",hue='Survived',data=data_all.iloc[:train_len,:])

png

对Embarked属性进行dummy操作

1	data_all=pd.get_dummies(data_all,columns=['Embarked'],prefix='Embarked')

剩下Ticket属性看起来比较复杂，先不做任何提取。

下面再继续看目前的data_all的信息。

1	data_all.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1309 entries, 0 to 1308
Data columns (total 23 columns):
Age                1309 non-null float64
Cabin              295 non-null object
Fare               1309 non-null float64
Name               1309 non-null object
Parch              1309 non-null int64
PassengerId        1309 non-null int64
Pclass             1309 non-null int64
Sex                1309 non-null int64
SibSp              1309 non-null int64
Survived           891 non-null float64
Ticket             1309 non-null object
Title              1309 non-null int64
Family_size        1309 non-null int64
Last_Name          1309 non-null object
Family_Survival    1309 non-null float64
FareBin            1309 non-null category
FareBin_Code       1309 non-null int64
AgeBin             1309 non-null category
AgeBin_Code        1309 non-null int64
Embarked_C         1309 non-null uint8
Embarked_None      1309 non-null uint8
Embarked_Q         1309 non-null uint8
Embarked_S         1309 non-null uint8
dtypes: category(2), float64(4), int64(9), object(4), uint8(4)
memory usage: 181.8+ KB

# 选用部分特征做为最终特征
data_all=data_all[['PassengerId','Pclass','Sex','Title','Family_size','Family_Survival','FareBin_Code','AgeBin_Code',
                  'Embarked_C','Embarked_None','Embarked_Q','Embarked_S','Survived']]
g = sns.heatmap(data_all.iloc[:train_len,:].drop(['PassengerId'],axis=1).corr(),cmap="BrBG",annot=True)

png

Scale以及模型选择

X_train_original=data_all.iloc[:train_len,:].drop(['PassengerId','Survived'],axis=1)
Y_train=data_all.iloc[:train_len,:]['Survived'].astype(int)
test_original=data_all.iloc[train_len:,:].drop(['PassengerId','Survived'],axis=1)
testID=data_all.iloc[train_len:,:]['PassengerId']

# nn using skorch 
# 神经网络模型
import torch
from torch import nn
import torch.nn.functional as F
torch.manual_seed(0)

class ClassifierModule(nn.Module):
    def __init__(
            self,
            num_units=20,
            nonlin=F.relu,
            dropout=0.5,
    ):
        super(ClassifierModule, self).__init__()
        self.num_units = num_units
        self.nonlin = nonlin
        self.dropout = dropout
        self.dense0 = nn.Linear(11, num_units)
        self.nonlin = nonlin
        self.dropout = nn.Dropout(dropout)
        self.dense1 = nn.Linear(num_units, 10)
        self.output = nn.Linear(10, 2)
    def forward(self, X, **kwargs):
        X = self.nonlin(self.dense0(X))
        X = self.dropout(X)
        X = F.relu(self.dense1(X))
        X = F.softmax(self.output(X), dim=-1)
        return X

from skorch import NeuralNetClassifier
from skorch.dataset import CVSplit
net = NeuralNetClassifier(
    ClassifierModule,
    max_epochs=500,
    lr=0.02,
    train_split=CVSplit(5),
#     device='cuda',  # uncomment this to train with CUDA
)
from sklearn.model_selection import GridSearchCV
X=X_train.astype(np.float32)
Y=Y_train.astype(np.int64)
params = {
    'lr': [0.02],
    'max_epochs': [500],
    'module__num_units': [20],
}
gs = GridSearchCV(net, params, refit=True,cv=2, scoring='accuracy')
gs.fit(X, Y)
print(gs.best_score_, gs.best_params_)
gs.estimator.set_params(**gs.best_params_).fit(X,Y)
IDtest=pd.read_csv('data/test.csv')['PassengerId']
test_Survived_nn=pd.Series(gs.predict(test.astype(np.float32)),name='Survived')
nn_results=pd.concat([IDtest,test_Survived_nn],axis=1)
nn_results.to_csv('nn_results.csv',index=False)
nn_results.head()

# scalar
scaler=MinMaxScaler()
X_train= scaler.fit_transform(X_train_original)
test= scaler.transform(test_original)

kfold = StratifiedKFold(n_splits=5)
# Modeling step Test differents algorithms
random_state = 2
classifiers = []
classifiers.append(SVC(random_state=random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state), random_state=random_state,
                                          learning_rate=0.1))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(MLPClassifier(random_state=random_state))
classifiers.append(KNeighborsClassifier())
classifiers.append(LogisticRegression(random_state=random_state))
classifiers.append(LinearDiscriminantAnalysis())
cv_results = []
for classifier in classifiers:
    cv_results.append(cross_val_score(classifier, X_train, y=Y_train, scoring="accuracy", cv=kfold))
cv_means = []
cv_std = []
for cv_result in cv_results:
    cv_means.append(cv_result.mean())
    cv_std.append(cv_result.std())
cv_res = pd.DataFrame(
        {"CrossValMeans": cv_means, "CrossValerrors": cv_std, "Algorithm": ["SVC", "DecisionTree", "AdaBoost","RandomForest", "ExtraTrees","GradientBoosting","MultipleLayerPerceptron", "KNeighboors","LogisticRegression","LinearDiscriminantAnalysis"]})
g = sns.barplot("CrossValMeans", "Algorithm", data=cv_res, palette="Set3", orient="h", **{'xerr': cv_std})
g.set_xlabel("Mean Accuracy")
g = g.set_title("Cross validation scores")

png

# Adaboost
DTC = DecisionTreeClassifier()
adaDTC = AdaBoostClassifier(DTC, random_state=7)
ada_param_grid = {"base_estimator__criterion": ["entropy"],
                      "base_estimator__splitter": [ "random"],
                      "algorithm": ["SAMME"],
                      "n_estimators": [100],
                      "learning_rate": [0.0001]}
gsadaDTC = GridSearchCV(adaDTC, param_grid=ada_param_grid, cv=kfold, scoring="accuracy", n_jobs=4, verbose=1)
gsadaDTC.fit(X_train, Y_train)
ada_best = gsadaDTC.best_estimator_
print('AdaBoost:')
print(gsadaDTC.best_params_)
print(gsadaDTC.best_score_)

Fitting 5 folds for each of 1 candidates, totalling 5 fits


[Parallel(n_jobs=4)]: Done   5 out of   5 | elapsed:    3.0s finished


AdaBoost:
{'algorithm': 'SAMME', 'base_estimator__criterion': 'entropy', 'base_estimator__splitter': 'random', 'learning_rate': 0.0001, 'n_estimators': 100}
0.8294051627384961

# ExtraTrees
ExtC = ExtraTreesClassifier()
## Search grid for optimal parameters
ex_param_grid = {"max_depth": [None],
                     "max_features": [0.2,0.3],
                     "min_samples_split": [3,5],
                     "min_samples_leaf": [5,],
                     "bootstrap": [False],
                     "n_estimators": [500],
                     "criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC, param_grid=ex_param_grid, cv=kfold, scoring="accuracy", n_jobs=4, verbose=1)
gsExtC.fit(X_train, Y_train)
print('Extra Tree:')
ExtC_best = gsExtC.best_estimator_
print(gsExtC.best_params_)
# Best score
print(gsExtC.best_score_)

Fitting 5 folds for each of 4 candidates, totalling 20 fits


[Parallel(n_jobs=4)]: Done  20 out of  20 | elapsed:    8.1s finished


Extra Tree:
{'bootstrap': False, 'criterion': 'gini', 'max_depth': None, 'max_features': 0.3, 'min_samples_leaf': 5, 'min_samples_split': 5, 'n_estimators': 500}
0.8462401795735129

#xgboost
XGBC = xgboost.sklearn.XGBClassifier()
xgbc_param_grid = {
                     "n_estimators": [500],
                    'learning_rate':[0.001],
                    'max_depth':[3],
                    'booster':['gbtree']}
gsXGBC= GridSearchCV(XGBC, param_grid=xgbc_param_grid, cv=kfold, scoring="accuracy", n_jobs=4, verbose=1)
gsXGBC.fit(X_train, Y_train)
print('xgboost:')
XGBC_best = gsXGBC.best_estimator_
print(gsXGBC.best_params_)
print(gsXGBC.best_score_)

Fitting 5 folds for each of 1 candidates, totalling 5 fits


[Parallel(n_jobs=4)]: Done   5 out of   5 | elapsed:    2.5s finished


xgboost:
{'booster': 'gbtree', 'learning_rate': 0.001, 'max_depth': 3, 'n_estimators': 500}
0.8518518518518519

LR=LogisticRegression()
lr_param_grid = {
       'penalty':['l1'],
        'tol':[1e-4],
        'C':[10]}
gsLR = GridSearchCV(LR, param_grid=lr_param_grid, cv=kfold, scoring="accuracy", n_jobs=4, verbose=1)
gsLR.fit(X_train, Y_train)
print('Logistic Regression:')
LR_best = gsLR.best_estimator_
print(gsLR.best_params_)
print(gsLR.best_score_)

Fitting 5 folds for each of 1 candidates, totalling 5 fits
Logistic Regression:
{'C': 10, 'penalty': 'l1', 'tol': 0.0001}
0.8361391694725028


[Parallel(n_jobs=4)]: Done   5 out of   5 | elapsed:    2.3s finished

# RFC Parameters tunning
RFC = RandomForestClassifier()
## Search grid for optimal parameters
rf_param_grid = {"max_depth": [None],
                "max_features": ['auto',0.3],
                "min_samples_split": [5],
                "min_samples_leaf": [6],
                "bootstrap": [False],
                "n_estimators": [150],
                "criterion": ["gini"]}
gsRFC = GridSearchCV(RFC, param_grid=rf_param_grid, cv=kfold, scoring="accuracy", n_jobs=4, verbose=1)
gsRFC.fit(X_train, Y_train)
print('Random Forest:')
RFC_best = gsRFC.best_estimator_
print(gsRFC.best_params_)
# Best score
print(gsRFC.best_score_)

Fitting 5 folds for each of 2 candidates, totalling 10 fits


[Parallel(n_jobs=4)]: Done  10 out of  10 | elapsed:    3.1s finished


Random Forest:
{'bootstrap': False, 'criterion': 'gini', 'max_depth': None, 'max_features': 0.3, 'min_samples_leaf': 6, 'min_samples_split': 5, 'n_estimators': 150}
0.8552188552188552

# Gradient boosting tunning
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss': ["deviance"],
                     'n_estimators': [200],
                     'learning_rate': [0.01],
                     'max_depth': [4],
                     'min_samples_leaf': [100],
                     "max_features": ['auto']
                     }
gsGBC = GridSearchCV(GBC, param_grid=gb_param_grid, cv=kfold, scoring="accuracy", n_jobs=4, verbose=1)
gsGBC.fit(X_train, Y_train)
print('Gradient Boost:')
print(gsGBC.best_params_)
GBC_best = gsGBC.best_estimator_
print(gsGBC.best_score_)

Fitting 5 folds for each of 1 candidates, totalling 5 fits


[Parallel(n_jobs=4)]: Done   5 out of   5 | elapsed:    2.5s finished


Gradient Boost:
{'learning_rate': 0.01, 'loss': 'deviance', 'max_depth': 4, 'max_features': 'auto', 'min_samples_leaf': 100, 'n_estimators': 200}
0.8484848484848485

### SVC classifier
SVMC = SVC(probability=True)
svc_param_grid = {'kernel': ['rbf'],
                      'gamma': [ 1],
                      'C': [0.1]}
gsSVMC = GridSearchCV(SVMC, param_grid=svc_param_grid, cv=kfold, scoring="accuracy", n_jobs=4, verbose=1)
gsSVMC.fit(X_train, Y_train)
print('SVC')
print(gsSVMC.best_params_)
SVMC_best = gsSVMC.best_estimator_
# Best score
print(gsSVMC.best_score_)

Fitting 5 folds for each of 1 candidates, totalling 5 fits


[Parallel(n_jobs=4)]: Done   5 out of   5 | elapsed:    2.5s finished


SVC
{'C': 0.1, 'gamma': 1, 'kernel': 'rbf'}
0.8305274971941639

#Voting
votingC = VotingClassifier(estimators=[('rfc', RFC_best), ('extc', ExtC_best),
                                           ('svc', SVMC_best), ('adac', ada_best), ('gbc', GBC_best),
                                           ('xgbc',XGBC_best),('lr',LR_best)], voting='soft',
                               n_jobs=4)
votingC = votingC.fit(X_train, Y_train)
voting_score=cross_val_score(votingC, X_train, y=Y_train, scoring="accuracy", cv=kfold)
print('voting:',voting_score)

voting: [0.86592179 0.84357542 0.85393258 0.81460674 0.84745763]

#Stacking
stack = StackingClassifier(classifiers=[RFC_best, ExtC_best,SVMC_best,ada_best,GBC_best,LR_best],
                              meta_classifier=XGBC_best)
stack = stack.fit(X_train, Y_train)
stack_score=cross_val_score(stack,X_train, y=Y_train, scoring="accuracy", cv=kfold)
print('stack_score:',stack_score)

stack_score: [0.83240223 0.81564246 0.85393258 0.82022472 0.82485876]

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5)):
        """Generate a simple plot of the test and training learning curve"""
        plt.figure()
        plt.title(title)
        if ylim is not None:
            plt.ylim(*ylim)
        plt.xlabel("Training examples")
        plt.ylabel("Score")
        train_sizes, train_scores, test_scores = learning_curve(
            estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
        train_scores_mean = np.mean(train_scores, axis=1)
        train_scores_std = np.std(train_scores, axis=1)
        test_scores_mean = np.mean(test_scores, axis=1)
        test_scores_std = np.std(test_scores, axis=1)
        plt.grid()
        plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                         train_scores_mean + train_scores_std, alpha=0.1,
                         color="r")
        plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                         test_scores_mean + test_scores_std, alpha=0.1, color="g")
        plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
                 label="Training score")
        plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
                 label="Cross-validation score")
        plt.legend(loc="best")
        return plt
g = plot_learning_curve(gsRFC.best_estimator_, "RF mearning curves", X_train, Y_train, cv=kfold)
g = plot_learning_curve(gsExtC.best_estimator_, "ExtraTrees learning curves", X_train, Y_train, cv=kfold)
g = plot_learning_curve(gsSVMC.best_estimator_, "SVC learning curves", X_train, Y_train, cv=kfold)
g = plot_learning_curve(gsadaDTC.best_estimator_, "AdaBoost learning curves", X_train, Y_train, cv=kfold)
g = plot_learning_curve(gsGBC.best_estimator_, "GradientBoosting learning curves", X_train, Y_train, cv=kfold)
g = plot_learning_curve(gsXGBC.best_estimator_,"xgboost learning curves",X_train,Y_train,cv=kfold)
g = plot_learning_curve(gsLR.best_estimator_,"Logistic Regression learning curves",X_train,Y_train,cv=kfold)

png

nrows =3
ncols=2
fig, axes = plt.subplots(nrows=nrows, ncols=ncols, sharex="all", figsize=(15, 15))
names_classifiers = [('rfc', RFC_best), ('extc', ExtC_best),
                        ('adac', ada_best), ('gbc', GBC_best),
                        ('xgbc',XGBC_best)]
nclassifier = 0
for row in range(nrows):
    for col in range(ncols):
        if nclassifier>=5:
            break
        name = names_classifiers[nclassifier][0]
        classifier = names_classifiers[nclassifier][1]
        indices = np.argsort(classifier.feature_importances_)[::-1][:40]
        g = sns.barplot(y=X_train_original.columns[indices][:40], x=classifier.feature_importances_[indices][:40],
                            orient='h', ax=axes[row][col])
        g.set_xlabel("Relative importance", fontsize=12)
        g.set_ylabel("Features", fontsize=12)
        g.tick_params(labelsize=9)
        g.set_title(name + " feature importance")
        nclassifier += 1
test_Survived_RFC = pd.Series(RFC_best.predict(test), name="RFC")
test_Survived_ExtC = pd.Series(ExtC_best.predict(test), name="ExtC")
test_Survived_SVMC = pd.Series(SVMC_best.predict(test), name="SVC")
test_Survived_AdaC = pd.Series(ada_best.predict(test), name="Ada")
test_Survived_GBC = pd.Series(GBC_best.predict(test), name="GBC")
test_Survived_XGBC=pd.Series(XGBC_best.predict(test),name='XGBC')
test_Survived_LR=pd.Series(LR_best.predict(test),name='LR')
test_Survived_Voting=pd.Series(votingC.predict(test),name='Voting')
    # Concatenate all classifier results
ensemble_results = pd.concat(
        [test_Survived_RFC, test_Survived_ExtC, test_Survived_AdaC, test_Survived_GBC, test_Survived_SVMC,
         test_Survived_XGBC,test_Survived_LR,test_Survived_Voting], axis=1)
g = sns.heatmap(ensemble_results.corr(), annot=True)

png

test_Survived = pd.Series(votingC.predict(test), name="Survived")
test_Survived_XGBC_submition=pd.Series(XGBC_best.predict(test),name='Survived')
xgbc_results=pd.concat([IDtest,test_Survived_XGBC_submition],axis=1)
xgbc_results.to_csv('xgbc.csv',index=False)
print(IDtest.shape)
print(test_Survived.shape)
results = pd.concat([IDtest, test_Survived], axis=1)
results.to_csv("ensemble_python_voting.csv", index=False)

(418,)
(418,)

最后提交的voting方法的结果得到了0.81339的分数,还有一定的提升空间，其中超参数调整应该还会带来一定的效果提升。

参考

1、yassineghouzam’s kernel
2、konstantinmasich’s kernel
3、pandas document
4、seaborn tutorial
5、notebook