import osimport moxing as moxif not os.path.exists('./dataset_2020.zip'):    mox.file.copy('obs://modelarts-labs-bj4/course/ai_in_action/2021/machine_learning/hard_drive_disk_fail_prediction/dataset_2020.zip', './dataset_2020.zip')    os.system('unzip dataset_2020.zip')
if not os.path.exists('./dataset_2020'):    raise Exception('错误！数据不存在！')
!ls -lh ./dataset_2020INFO:root:Using MoXing-v1.17.3-INFO:root:Using OBS-Python-SDK-3.20.7total 102M-rw-r--r-- 1 ma-user ma-group  51M Mar 21 11:56 2020-12-08.csv-rw-r--r-- 1 ma-user ma-group  51M Mar 21 11:56 2020-12-09.csv-rw-r--r-- 1 ma-user ma-group 1.2M Mar 21 11:55 dataset_2020.csv-rw-r--r-- 1 ma-user ma-group 3.5K Mar 22 15:59 prepare_data.py

import pandas as pddf_data = pd.read_csv("./dataset_2020/2020-12-08.csv")type(df_data)pandas.core.frame.DataFrame

print('单个csv文件数据的规模，行数：%d, 列数：%d' % (df_data.shape[0], df_data.shape[1]))单个csv文件数据的规模，行数：162008, 列数：149

df_data.head()

df_data.describe()

df_data.isnull().sum()date                         0serial_number                0model                        0capacity_bytes               0failure                      0smart_1_normalized         179smart_1_raw                179smart_2_normalized      103169smart_2_raw             103169smart_3_normalized        1261smart_3_raw               1261smart_4_normalized        1261smart_4_raw               1261smart_5_normalized        1221smart_5_raw               1221smart_7_normalized        1261smart_7_raw               1261smart_8_normalized      103169smart_8_raw             103169smart_9_normalized         179smart_9_raw                179smart_10_normalized       1261smart_10_raw              1261smart_11_normalized     161290smart_11_raw            161290smart_12_normalized        179smart_12_raw               179smart_13_normalized     161968smart_13_raw            161968smart_15_normalized     162008                         ...  smart_232_normalized    160966smart_232_raw           160966smart_233_normalized    160926smart_233_raw           160926smart_234_normalized    162008smart_234_raw           162008smart_235_normalized    160964smart_235_raw           160964smart_240_normalized     38968smart_240_raw            38968smart_241_normalized     56030smart_241_raw            56030smart_242_normalized     56032smart_242_raw            56032smart_245_normalized    161968smart_245_raw           161968smart_247_normalized    162006smart_247_raw           162006smart_248_normalized    162006smart_248_raw           162006smart_250_normalized    162008smart_250_raw           162008smart_251_normalized    162008smart_251_raw           162008smart_252_normalized    162008smart_252_raw           162008smart_254_normalized    161725smart_254_raw           161725smart_255_normalized    162008smart_255_raw           162008Length: 149, dtype: int64

import numpy as npimport matplotlib.pyplot as plt%matplotlib inline
df_data_null_num = df_data.isnull().sum()x = list(range(len(df_data_null_num)))y = df_data_null_num.valuesplt.plot(x, y)plt.show()

valid = df_data[df_data['failure'] == 0]failed = df_data[df_data['failure'] == 1]print("valid hdds:",len(valid))print("failed hdds:",len(failed))valid hdds: 162000failed hdds: 8

# if not os.path.exists('./dataset_2020/dataset_2020.csv'):#     os.system('python ./dataset_2020/prepare_data.py')import gcdel df_data   # 删除 df_data 对象gc.collect()  # 由于接下来的代码将加载日志数据到df_data对象中，为避免内存溢出的风险，可以在此处先手动回收内存，因为jupyterlab在运行过程中不会自动回收环境中的内存

df_data = pd.read_csv("./dataset_2020/dataset_2020.csv")valid = df_data[df_data['failure'] == 0]failed = df_data[df_data['failure'] == 1]print("valid hdds:", len(valid))print("failed hdds:", len(failed))valid hdds: 1497failed hdds: 1497

df_data.model.value_counts()ST12000NM0007                         664ST4000DM000                           491ST8000NM0055                          320ST12000NM0008                         293TOSHIBA MG07ACA14TA                   212ST8000DM002                           195HGST HMS5C4040BLE640                  193HGST HUH721212ALN604                  153TOSHIBA MQ01ABF050                     99ST12000NM001G                          53HGST HMS5C4040ALE640                   50ST500LM012 HN                          40TOSHIBA MQ01ABF050M                    35HGST HUH721212ALE600                   34ST10000NM0086                          29ST14000NM001G                          23HGST HUH721212ALE604                   21ST500LM030                             15HGST HUH728080ALE600                   14Seagate BarraCuda SSD ZA250CM10002     12WDC WD5000LPVX                         11WDC WUH721414ALE6L4                    10ST6000DX000                             9TOSHIBA MD04ABA400V                     3ST8000DM004                             2ST18000NM000J                           2Seagate SSD                             2ST4000DM005                             2ST8000DM005                             1ST16000NM001G                           1DELLBOSS VD                             1TOSHIBA HDWF180                         1HGST HDS5C4040ALE630                    1HGST HUS726040ALE610                    1WDC WD5000LPCX                          1Name: model, dtype: int64

df_data_model = df_data[df_data['model'] == 'ST12000NM0007']

features_specified = []features = [5, 9, 187, 188, 193, 194, 197, 198, 241, 242]for feature in features:    features_specified += ["smart_{0}_raw".format(feature)]X_data = df_data_model[features_specified]Y_data = df_data_model['failure']X_data.isnull().sum()smart_5_raw      1smart_9_raw      1smart_187_raw    1smart_188_raw    1smart_193_raw    1smart_194_raw    1smart_197_raw    1smart_198_raw    1smart_241_raw    1smart_242_raw    1dtype: int64

X_data = X_data.fillna(0) print("valid hdds:", len(Y_data) - np.sum(Y_data.values))print("failed hdds:", np.sum(Y_data.values))valid hdds: 325failed hdds: 339

from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(X_data, Y_data, test_size=0.2, random_state=0) 

from sklearn.ensemble import RandomForestClassifier 
rfc = RandomForestClassifier()

rfc.fit(X_train, Y_train)/home/ma-user/anaconda3/envs/XGBoost-Sklearn/lib/python3.6/site-packages/sklearn/ensemble/forest.py:248: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.  "10 in version 0.20 to 100 in 0.22.", FutureWarning)
RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',            max_depth=None, max_features='auto', max_leaf_nodes=None,            min_impurity_decrease=0.0, min_impurity_split=None,            min_samples_leaf=1, min_samples_split=2,            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=None,            oob_score=False, random_state=None, verbose=0,            warm_start=False)

Y_pred = rfc.predict(X_test) 

from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
print("Model used is: Random Forest classifier")   acc = accuracy_score(Y_test, Y_pred) print("The accuracy is {}".format(acc))   prec = precision_score(Y_test, Y_pred) print("The precision is {}".format(prec))   rec = recall_score(Y_test, Y_pred) print("The recall is {}".format(rec))   f1 = f1_score(Y_test, Y_pred) print("The F1-Score is {}".format(f1)) Model used is: Random Forest classifierThe accuracy is 0.849624060150376The precision is 0.9122807017543859The recall is 0.7761194029850746The F1-Score is 0.8387096774193548

import picklewith open('hdd_failure_pred.pkl', 'wb') as fw:    pickle.dump(rfc, fw)

with open('hdd_failure_pred.pkl', 'rb') as fr:    new_rfc = pickle.load(fr)

new_Y_pred = new_rfc.predict(X_test)new_prec = precision_score(Y_test, new_Y_pred)print("The precision is {}".format(new_prec))The precision is 0.9122807017543859

import seaborn as snsimport matplotlib.pyplot as pltfrom sklearn.metrics import confusion_matrix 
LABELS = ['Healthy', 'Failed'] conf_matrix = confusion_matrix(Y_test, Y_pred) plt.figure(figsize =(6, 6)) sns.heatmap(conf_matrix, xticklabels = LABELS,              yticklabels = LABELS, annot = True, fmt ="d"); plt.title("Confusion matrix") plt.ylabel('True class') plt.xlabel('Predicted class') plt.show()