다중 분류 원리

# soft max and cross entropy

import numpy as np
from sklearn.datasets import load_iris

# data 준비
iris=load_iris()

X=iris.data
y=iris.target
# one hot incoding
onehot_y=np.eye(3)[y]

# 초기화
w=np.random.randn(4,3)
b=np.random.randn(3)

# learning rate
lr=0.001


def softmax(t):
    return np.exp(t)/np.sum(np.exp(t),axis=1).reshape(-1,1)


pred_y=softmax(X@w+b)

cross_entropy = -(onehot_y*np.log(pred_y+1e-7)).sum()/len(y)


log=[cross_entropy]


for i in range(30000):
    
    # 현재의 w,b값에 따른 예측(기준 위치)
    pred_y=softmax(X@w+b)
    
    # w에 관한 기울기 * lr
    dw = (X.T@(onehot_y-pred_y))/-len(y)*lr
    
    # b에 관한 기울기 * lr
    db = (onehot_y-pred_y).sum()/-len(y)*lr
    
    # w, b 값 변화
    w-=dw
    b-=db
    
    # cost 측정 및 수집
    cross_entropy = -(onehot_y*np.log(pred_y+1e-7)).sum()/len(y)
    log.append(cross_entropy)
    if i%1000==0:print('%5d    '%(i),cross_entropy)

    0     9.305923777318322
 1000     1.1537510254481753
 2000     1.035036966529715
 3000     0.9362014445714768
 4000     0.8506820559529423
 5000     0.7756919367027773
 6000     0.7095581752035464
 7000     0.6511145080902988
 8000     0.5994513003489882
 9000     0.55380705828736
10000     0.5135197083443904
11000     0.47800295842371004
12000     0.44673239902485384
13000     0.41923482640058474
14000     0.3950786815756025
15000     0.37386593684666247
16000     0.35522695997982245
17000     0.33881965670814407
18000     0.324332496567667
19000     0.31148898033007555
20000     0.3000504588166544
21000     0.2898155344894717
22000     0.280616336415144
23000     0.2723131992459631
24000     0.2647892768634723
25000     0.2579459601338459
26000     0.25169931763885917
27000     0.24597742144465964
28000     0.2407183229487914
29000     0.23586847655614296

 

plt.plot(log)

plt.plot(log[:3000])

# logisticRegression 겨로가 값과 비교( cost 계산 내용이 달라 결과는 다르다 )

from sklearn.linear_model import LogisticRegression

model=LogisticRegression()
model.fit(X,y)
model.coef_,w,model.intercept_,b

(array([[ 0.41498833,  1.46129739, -2.26214118, -1.0290951 ],
        [ 0.41663969, -1.60083319,  0.57765763, -1.38553843],
        [-1.70752515, -1.53426834,  2.47097168,  2.55538211]]),
 array([[ 0.33991183,  1.54167625, -0.32286789],
        [ 2.01092954, -1.09940909, -1.25881242],
        [-0.82842195,  1.10793973,  2.37857752],
        [-1.75396921, -1.50312094,  1.39323349]]),
 array([ 0.26560617,  1.08542374, -1.21471458]),
 array([ 1.12281412, -1.40656273, -0.1876931 ]))

# score
(np.argmax(pred_y,axis=1)==iris.target).mean()

0.9466666666666667