前処理

一般例

import pandas as pd

import numpy as np

from matplotlib import pyplot as plt

%matplotlib inline

data = pd.read_csv('xxx.csv', header=None)

len(data)

data.head(10)

data.tail()

row, col = data.shape

data.describe()

data['y'].describe()

data.info()

age_bining = pd.cut(data['age'], [0,20,30,40,50])

age = pd.crosstab(age_bining, data['y'], margins=True)

age['rate'] = age[1] / age['All']

data['year'] = data['datetime'].apply(lambda x: x.split('-')[0])

data['year'] = data['year'].astype(np.int)

# 欠損値

data.isnull().any()

data.isnull().sum()

data.fillna(0)

data['kcal'].fillna(data['kcal'].mean())

data.dropna(subset=['kcal'])

data['kcal'].value_counts()

# 相関関係

data[['y', 'week']].corr()

data['y'].mean()

data['y'].median()

data[data['y'] > = 100]

data[data['week'] == 'MON'].sort_values(by='y', ascending=False)

data[data['week'] == 'MON']['y'].mean()

data[data['temp'] >= data['temp'].mean()]

data[['y', 'week']].boxplot(by='week')

data.plot.scatter(x='kcal', y='y', figsize=(5, 5))

# または

plt.scatter(data['kcal'], data['y'], label='xxx')

plt.plot(data['kcal'], data['y'], label='xxx', color='red')

plt.legend()

plt.show()

ax = data['y'].plot(figsize=(12, 4), title='Graph')

ax.set_xlabel('time')

ax.set_ylabel('y')

plt.axvline(x=data['y'].mean(), color='red')

data['y'].plot.hist(figsize=(12, 4), title='Graph', grid=True)

plt.savefig('sample_fig.png')

train['pred'] = pred

train['res'] = train['y'] - train['pred']

train.sort_values(by='res')

★乱数

np.random().seed(0)

num = np.random().rand(5) # 一様乱数

num = np.random().randn(5) # 正規分布に従う乱数

num = np.random().binomial(100, 0.5, size=(5)) # 二項分布に従う乱数

df = pd.DataFrame(np.random().rand(10, 4))

b_list = np.random().choice(a_list, 5)

★CSV関連

df = pd.read_csv("xxx.data", header=None)

df.columns = ["aaa", "bbb", ...]

import csv

with open("xxx.csv", "w") as csvfile:

writer = csv.writer(csvfile, lineterminator="¥n")

writer.writerow(["aaa", "bbb", ...])

data = {

"city": ["aaa", "bbb"],

"year": [1998, 2000]

}

df = pd.DataFrame(data)

df.to_csv("xxx.csv")

★3σ法

col = 'y'

mean = df.mean()

sigma = df.std()

low = mean[col] - 3 * sigma[col]

high = mean[col] + 3 * sigma[col]

df2 = df[(df[col] > low) & (df[col] < high)]

cols = df.columns

_df = df

for col in cols:

low = mean[col] - 3 * sigma[col]

high = mean[col] + 3 * sigma[col]

_df = _df[(_df[col] > low) & (_df[col] < high)]

★欠損値処理

df.replace('None', np.nan, inplace=True)

df.dropna(subset=['thickness'], inplace=True)

df['thickness'].fillna(1, inplace=True)

df.replace('None', np.nan, inplace=True)

df['thickness'] = df['thickness'].astype('float64')

thickness_mean = df['thickness'].mean()

df['thickness'].fillna(thickness_mean, inplace=True)

基礎集計

・各種代表値（平均、分散、標準偏差など）を計算

・散布図行列をプロットして傾向を調べる

・相関行列を表示して傾向を調べる

前処理

・欠損値の処理、補間（空欄のあるデータを消去・補間）

・名寄せ（表記の揺れを統一）

・正規化や標準化

・時系列データの時間粒度の変更

・画像データのサイズ統一化、グレースケール化

・テキストデータの分かち書き、不要文字の削除、半角全角の統一

・データの不均衡を解消するのには、imbalanced-learnとかSMOTEとか利用

Stop Feeding Garbage To Your Model

★EDA（Exploratory Data Analysis）

統計量

・各カラムの型や値の分布、欠損値、外れ値

・目的関数と各変数の相関や関係性

・変数の平均、標準偏差、最大、最小、分位点

・カテゴリ変数の値の種類数

・変数の欠損値の数

・変数間の相関係数

可視化手法

・棒グラフ

・箱ひげ図、バイオリンプロット

・散布図

・折れ線グラフ

・ヒートマップ

・ヒストグラム

・Q-Qプロット

・t-SNE、UMAP

データ構造を対象とした前処理：抽出、集約、結合、分割、生成、展開

データ内容を対象とした前処理：数値、カテゴリ、日時、文字、位置情報

selection

df.iloc[:, 0:6] # not good

df[['reserve_id', 'hotel_id']]

df.loc[:, ['reserve_id', 'hotel_id']]

df.drop(['people_num', 'total_price'], axis=1, inplace=True) # 列の削除

df[(df['date'] >= '2018-12-24') & (df['date'] <= '2018-12-30')] # not good

df.loc[(df['date'] >= '2018-12-24') & (df['date'] <= '2018-12-30'), :] # not good

df.query('"2018-12-24" <= date <= "2018-12-30"')

df.sample(frac=0.5) # 50%サンプリング

target = pd.Series(df['customer_id'].unique()).sample(frac=0.5)

df[df['customer_id'].isin(target)]

aggregation

result = df \

.groupby('hotel_id') \

.agg({'reserve_id': 'count', 'customer_id': 'nunique'})

result.reset_index(inplace=True)

result.columns = ['hotel_id', 'rsv_cnt', 'cus_cnt']

result = df \

.groupby(['hotel_id', 'people_num'])['total_price'] \

.sum().reset_index()

result.rename(columns={'total_price': 'price_sum'}, inplace=True)

result = df \

.groupby('hotel_id') \

.agg({'total_price': ['max', 'min', 'mean', 'median',

lambda x: np.percentile(x, q=20)]}) \

.reset_index()

result.columns = ['hotel_id', 'price_max', 'price_min', 'price_mean',

'price_median', 'price_20per']

result = df \

.groupby('hotel_id') \

.agg({'total_price': ['var', 'std']}).reset_index()

result.columns = ['hotel_id', 'price_var', 'price_std']

result.fillna(0, inplace=True)

df['total_price'].round(-3).mode() # 四捨五入した後に最頻値を算出

df['reserve_datetime'] = pd.to_datetime(

df['reserve_datetime'], format='%Y-%m-%d %H:%M:%S'

)

df['log_no'] = df \

.groupby('customer_id')['reserve_datetime'] \

.rank(ascending=True, method='first') # 昇順

split

from sklearn.model_selection import train_test_split

from sklearn.model_selection import KFold

train_data, test_data, train_target, test_target = \

train_test_split(df.drop('fault_flg', axis=1),

df[['fault_flg']],

test_size=0.2)

train_data.reset_index(inplace=True, drop=True)

test_data.reset_index(inplace=True, drop=True)

train_target.reset_index(inplace=True, drop=True)

test_target.reset_index(inplace=True, drop=True)

row_no_list = list(range(len(train_target)))

# 交差検証用のデータ分割

k_fold = KFold(n_splits=4, shuffle=True)

for train_cv_no, test_cv_no in k_fold.split(row_no_list):

train_cv = train_data.iloc[train_cv_no, :]

test_cv = train_data.iloc[test_cv_no, :]

...