©2020-2024 ioDraw All rights reserved,
Privacy Policy
特征工程
1、 导入包并读取数据
import pandas as pd import numpy as np import matplotlib.pyplot as plt import
seabornas sns import datetime from tqdm import tqdm from sklearn.preprocessing
import LabelEncoder from sklearn.feature_selection import SelectKBest from
sklearn.feature_selection import chi2 from sklearn.preprocessing import
MinMaxScalerimport xgboost as xgb import lightgbm as lgb from catboost import
CatBoostRegressorimport warnings from sklearn.model_selection import
StratifiedKFold, KFold from sklearn.metrics import accuracy_score, f1_score,
roc_auc_score, log_loss warnings.filterwarnings('ignore') #读取文件 data_train=pd.
read_csv('tianchi/train.csv') data_test_a=pd.read_csv('tianchi/testA.csv')
2、特征预处理
数据EDA部分我们已经对数据的大概和某些特征分布有了了解，数据预处理部分一般我们要处理一些EDA阶段分析出来的问题，这里介绍了数据缺失值的填充，时间格式特征的转化处理，某些对象类别特征的处理。
首先我们查找出数据中的对象特征和数值特征
numerical_fea = list(data_train.select_dtypes(exclude=['object']).columns)
category_fea= list(filter(lambda x: x not in numerical_fea,list(data_train.
columns))) label = 'isDefault' numerical_fea.remove(label)
缺失值填充
把所有缺失值替换为指定的值0
data_train = data_train.fillna(0)
向用缺失值上面的值替换缺失值
data_train = data_train.fillna(axis=0,method=‘ffill’)
纵向用缺失值下面的值替换缺失值,且设置最多只填充两个连续的缺失值
data_train = data_train.fillna(axis=0,method=‘bfill’,limit=2)
#查看缺失值情况 data_train.isnull().sum() #按照平均数填充数值型特征 data_train[numerical_fea] =
data_train[numerical_fea].fillna(data_train[numerical_fea].median()) data_test_a
[numerical_fea] = data_test_a[numerical_fea].fillna(data_train[numerical_fea].
median()) #按照众数填充类别型特征 data_train[category_fea] = data_train[category_fea].
fillna(data_train[category_fea].mode()) data_test_a[category_fea] = data_test_a[
category_fea].fillna(data_train[category_fea].mode())
时间格式处理
#转化成时间格式 for data in [data_train, data_test_a]: data['issueDate'] = pd.
to_datetime(data['issueDate'],format='%Y-%m-%d') startdate = datetime.datetime.
strptime('2007-06-01', '%Y-%m-%d') #构造时间特征 data['issueDateDT'] = data[
'issueDate'].apply(lambda x: x-startdate).dt.days data_train['employmentLength']
.value_counts(dropna=False).sort_index()
对象类型特征转换到数值
def employmentLength_to_int(s): if pd.isnull(s): return s else: return np.int8(
s.split()[0]) for data in [data_train, data_test_a]: data['employmentLength'].
replace(to_replace='10+ years', value='10 years', inplace=True) data[
'employmentLength'].replace('< 1 year', '0 years', inplace=True) data[
'employmentLength'] = data['employmentLength'].apply(employmentLength_to_int)
data['employmentLength'].value_counts(dropna=False).sort_index()