import numpy as npimport pandas as npimport matplotlib.pyplot as pltimport seaborn as snsfrom scipy import stats
import warningswarnings.filterwarnings('ignore')%matplotlib inline

train_data_file = './zhengqi_train.txt'test_data_file = './zhengqi_test.txt'
train_data = pd.read_csv(train_data_file,sep='\t',encoding='utf-8') # 读取的是文本文件，位置分隔符‘\t’test_data = pd.read_csv(test_data_file,sep='\t'.encoding='utf-8')

# 数值类型numerical_feature = list(train.select_dtypes(exclude=['object']).columns)
len(numerical_feature)
# 连续型变量serial_feature = []# 离散型变量discrete_feature = []# 单值变量unique_feature = []
for fea in numerical_feature:  temp = train[fea].nuique() # 返回唯一值的个数  if temp==1:    unique_feature.append(fea)  elif temp <= 10:    discrete_feature.append(fea)  else:    serial_feature.append(fea)

train_data.info() # 查看字段的数据类型，有无缺失值test_data.info()
train_data.describe() # 查看每个字段的统计信息test_data.describe()
train_data.head() # 查看字段的具体信息test_data.head()

fig = plt.figure() # 创建一个实例ax = fig.add_subplots(2,1,1) # 创建Axes实例	ax.plot() # 

fig = plt.figure(figsize=(4,6)) # 指定绘图对象的宽度和高度sns.boxplot(train_data['V0'],orient='v',width=0.5) # 画一个字段的数据分布
### 多个字段的数据分布column = train_data.columns.tolist()[:39] # 列表头fig = plt.figure(figsize=(80,60),dpi=75) # 指定高度和宽度for i in range(38):  plt.subplot(7,8,i+1) # 7行8列子图  sns.boxplot(train_data[column[i]],orient='v',width=0.5) # 箱型图  plt.ylabel(column[i],fontsize=36)plt.show()

# 直方图和QQ图plt.figure(figsize=(10,5))
ax = plt.subplot(1,2,1)sns.distplot(train_data['V0'],fit=stats.norm)ax = plt.subplot(1,2,2)res = stats.probplot(train_data['V0'],plot=plt)
# 多个字段的直方图和QQ图train_cols = 6train_rows = len(train_data.columns)plt.figure(figsize=(4*train_cols,4*train_rows))
i=0for col in train_data.columns:  i+=1  ax = plt.subplot(train_rows,train_cols,i)  sns.distplot(train_data[col],fit=stats.norm)    i+=1  ax = plt.subplots(train_rows,train_cols,i)  res = stats.probplot(train_data[col],plot=plt)plt.tight_layout()plt.show()  

# 单个字段的KDE分布图plt.figure(figsize=(8,4),dpi=0)ax = sns.kdeplot(train_data['V0'],color='Red',shade=True)ax = sns.kdeplot(test_data['V0'],color='Blue',shade=True)ax.set_xlabel('V0')ax.set_ylabel('Frenquency')ax = ax.legend(['train','test'])
# 多个字段的KDE分布图dist_cols = 6dist_rows = len(train_data.columns)plt.figure(figsize=(4*dist_cols,4*dist_rows))i = 1for col in train_data.columns:  ax = plt.subplot(dist_rows,dist_cols,i)  ax = sns.kdeplot(train_data[col],color='Red',shade=True)	ax = sns.kdeplot(test_data[col],color='Blue',shade=True)  ax.set_xlabel(col)  ax.set_ylabel('Frenqucy')  ax = ax.legend(['train','test'])  i += 1plt.show()

data_train1 = train_data.drop(['V0','V1'],axis=1)train_corr = data_train1.corr() # 相关系数矩阵

ax = plt.subplots(figsize=(20,16))ax = sns.heatmap(train_corr,vmax=.8,square=True,annot=True) # 画热力图

k = 10cols = train_corr.nlargest(10,'target')['target'].index
cm = np.corrcoef(train_data[cols].values.T)hm = plt.subplots(figsize=(10,10)) # 调整画布大小hm = sns.heatmap(train_data[cols].corr(),annot=True,square=True)plt.show()

threshold = 0.5
corrmat = train_data.corr()top_corr_features = corrmat.index[abs(corrmat['target']) > threshold]plt.figure(figsize = (10,10))g = sns.heatmap(train_data[top_corr_features].corr(),annot=True,cmap='RdYlGn')

from sklearn.preprocessing import StandardScalerfrom sklearn.datasets import load_irisiris = load_iris()StandardScaler().fit_transform(iris.data)

from sklearn.preprocessing import MinMaxScalerMinMaxScaler().fit_transform(iris.data)

from sklearn.preprocessing import NormalizerNormalizar().fit_transform(iris.data)

from sklearn.preprocessing import BinarizerBinarizer(threshold).fit_transform(iris.data)

from sklearn.preprocessing import OneHotEncoderOneHotEncoder(categories='auto').fit_transform(iris.target.reshape((-1,1)))