머신러닝 - Deep Learning from scratch2019년 3월 23일

M.L (p.175)

import numpy as np

class Affine:
    def __init__(self,W,b):
        self.W = W
        self.b = b
        self.x = None
        self.dW = None
        self.db = None

    def forward(self,x):
        self.x = x
        out = np.dot(x,self.W) + self.b

        return out

    def backward(self,dout):
        dx = np.dot(dout,self.W.T)
        self.dW = np.dot(self.x.T,dout)
        self.db = np.sum(dout, axis=0)

        return dx

import numpy as np

class Affine:

def __init__(self,W,b):

self.W = W

self.b = b

self.x = None

self.dW = None

self.db = None

def forward(self,x):

self.x = x

out = np.dot(x,self.W) + self.b

return out

def backward(self,dout):

dx = np.dot(dout,self.W.T)

self.dW = np.dot(self.x.T,dout)

self.db = np.sum(dout, axis=0)

return dx

Affine 계층에 대한 순전파와 역전파에 대한 코드이다.

머신러닝 - Deep Learning from scratch2019년 3월 23일

M.L (p.174)

import numpy as np

X_dot_W = np.array([[0,0,0], [10,10,10]])
B = np.array([1,2,3])
print(X_dot_W)
print(B)
result = X_dot_W + B
print(result)

dY = np.array([[1,2,3], [4,5,6]])
print(dY)
dB = np.sum(dY,axis=0)
print(dB)

import numpy as np

X_dot_W = np.array([[0,0,0], [10,10,10]])

B = np.array([1,2,3])

print(X_dot_W)

print(B)

result = X_dot_W + B

print(result)

dY = np.array([[1,2,3], [4,5,6]])

print(dY)

dB = np.sum(dY,axis=0)

print(dB)

Affine 계층에서 bias 에 대한 연산에 대한 코드이해다. 순전파에선 XW + b 연산이 수행되는데 python 에선 broad cast 기능때문에 행렬의 형태를 맞추어 주지 않아도…

머신러닝 - Deep Learning from scratch2019년 3월 21일

M.L (p.170)

Affine transformation 신경망에서의 행렬곱에 대하여 주요 문장 ( ” 편미분 과정에서 적당히 행렬을 맞춰준다 “) 대응하는 차원의 원소 수를 일치시킨다. 끝에 알 수 없는 라운드…

머신러닝 - Deep Learning from scratch2019년 3월 20일

M.L (p.167)

Sigmoid function 의 순전파와 역전파를 계산그래프로 이해해보겠다. sigmoid 식이다. 함수 그래프 순전파 식 전개 역전파 전개 : ‘∂(’라운드 디‘라고 읽음) 편의 상 ‘라운드’ 라…

머신러닝 - Deep Learning from scratch2019년 3월 20일

M.L (p.166)

import numpy as np

class Relu:
    def __init__(self):
        self.mask = None

    def forward(self,x):
        self.mask = (x <= 0)
        out = x.copy()
        out[self.mask] = 0

        return out

    def backward(self,dout):
        dout[self.mask] = 0
        dx = dout

        return dx

x = np.array([[1.0, -0.5], [-2.0, 3.0]])
print(x)

mask = (x <= 0)
print(mask)
out = x.copy()
out[mask] = 0
print(out)

import numpy as np

class Relu:

def __init__(self):

self.mask = None

def forward(self,x):

self.mask = (x <= 0)

out = x.copy()

out[self.mask] = 0

return out

def backward(self,dout):

dout[self.mask] = 0

dx = dout

return dx

x = np.array([[1.0, -0.5], [-2.0, 3.0]])

print(x)

mask = (x <= 0)

print(mask)

out = x.copy()

out[mask] = 0

print(out)

ReLu function 을 numpy array로 순전파와 역전파를 구현하는 코드이다. ReLu의 식이며, x > 0 일때는 x 자체를 0보다 작을때는 0을 출력하는 function 이다. 역전파에…

머신러닝 - Deep Learning from scratch2019년 3월 20일

M.L (p.164)

import numpy as np

class MulLayer:
    def __init__(self):
        self.x = None
        self.y = None

    def forward(self,x,y):
        self.x = x
        self.y = y
        out = x * y

        return out

    def backward(self,dout):
        dx = dout * self.y
        dy = dout * self.x
        return dx,dy

class AddLayer:
    def __init__(self):
        pass

    def forward(self,x,y):
        out = x + y
        return out

    def backward(self,dout):
        dx = dout
        dy = dout
        return dx,dy

#variable
apple_price = 100
apple_num = 2
orange_price = 150
orange_num = 3
tax = 1.1

apple_mul_layer = MulLayer()
orange_mul_layer = MulLayer()
tax_mul_layer = MulLayer()
fruit_add_layer = AddLayer()

#forward
apple_total = apple_mul_layer.forward(apple_price,apple_num)
print(apple_total)
orange_total = orange_mul_layer.forward(orange_price,orange_num)
print(orange_total)
fruit_total = fruit_add_layer.forward(apple_total,orange_total)
print(fruit_total)
fruit_tax = tax_mul_layer.forward(fruit_total,tax)
print(fruit_tax)

#backward
d_total_price = 1
d_fruit_total, d_tax = tax_mul_layer.backward(d_total_price)
print(d_fruit_total,d_tax)
d_apple_total, d_orange_total = fruit_add_layer.backward(d_fruit_total)
print(d_apple_total,d_orange_total)
d_apple_price, d_apple_num = apple_mul_layer.backward(d_apple_total)
print(d_apple_price,d_apple_num)
d_orange_price, d_orange_num = orange_mul_layer.backward(d_orange_total)
print(d_orange_price,d_orange_num)

import numpy as np

class MulLayer:

def __init__(self):

self.x = None

self.y = None

def forward(self,x,y):

self.x = x

self.y = y

out = x * y

return out

def backward(self,dout):

dx = dout * self.y

dy = dout * self.x

return dx,dy

class AddLayer:

def __init__(self):

pass

def forward(self,x,y):

out = x + y

return out

def backward(self,dout):

dx = dout

dy = dout

return dx,dy

#variable

apple_price = 100

apple_num = 2

orange_price = 150

orange_num = 3

tax = 1.1

apple_mul_layer = MulLayer()

orange_mul_layer = MulLayer()

tax_mul_layer = MulLayer()

fruit_add_layer = AddLayer()

#forward

apple_total = apple_mul_layer.forward(apple_price,apple_num)

print(apple_total)

orange_total = orange_mul_layer.forward(orange_price,orange_num)

print(orange_total)

fruit_total = fruit_add_layer.forward(apple_total,orange_total)

print(fruit_total)

fruit_tax = tax_mul_layer.forward(fruit_total,tax)

print(fruit_tax)

#backward

d_total_price = 1

d_fruit_total, d_tax = tax_mul_layer.backward(d_total_price)

print(d_fruit_total,d_tax)

d_apple_total, d_orange_total = fruit_add_layer.backward(d_fruit_total)

print(d_apple_total,d_orange_total)

d_apple_price, d_apple_num = apple_mul_layer.backward(d_apple_total)

print(d_apple_price,d_apple_num)

d_orange_price, d_orange_num = orange_mul_layer.backward(d_orange_total)

print(d_orange_price,d_orange_num)

다음은 사과 2개와 귤 3개 소비세를 포함한 총 가격이 역전파를 통해 각각의 변수가 얼마나 총 가격에 얼마나 영향을 미치는지 파악해 보는 코드이다. 구현한 코드가…

머신러닝 - Deep Learning from scratch2019년 3월 20일

M.L (p.161)

import numpy as np

class MulLayer:
    def __init__(self):
        self.x = None
        self.y = None

    def forward(self,x,y):
        self.x = x
        self.y = y
        out = x * y

        return out

    def backward(self,dout):
        dx = dout * self.y
        dy = dout * self.x
        return dx,dy

class AddLayer:
    def __init__(self):
        pass

    def forward(self,x,y):
        out = x + y
        return out

    def backward(self,dout):
        dx = dout
        dy = dout
        return dx,dy

import numpy as np

class MulLayer:

def __init__(self):

self.x = None

self.y = None

def forward(self,x,y):

self.x = x

self.y = y

out = x * y

return out

def backward(self,dout):

dx = dout * self.y

dy = dout * self.x

return dx,dy

class AddLayer:

def __init__(self):

pass

def forward(self,x,y):

out = x + y

return out

def backward(self,dout):

dx = dout

dy = dout

return dx,dy

backpropagation(역전파)에 덧셈과 곱셈 노드에 대한 코드이다. 역전파는 계산그래프를 거꾸로 돌아가며 각 weight와 parameter가 Y(output value)에 얼마나 영향을 미치는지(미분을 통해) chain rule 이라는 성질을 이용해…

머신러닝 - Deep Learning from scratch2019년 3월 17일

M.L (p.143)

import numpy as np
from p88func import *
import sys,os
sys.path.append(os.pardir)
from mnist import load_mnist

(x_train, t_train),(x_test,t_test) = load_mnist(normalize= True, one_hot_label=True)

network = TwoLayerNet(input_size = 784, hidden_size = 50, output_size = 10)

iters_num = 10000
train_size = x_train.shape[0] #60,000
batch_size = 100
learning_rate = 0.1

train_loss_list = []
train_acc_list = []
test_acc_list = []

iter_per_epoch = max(train_size / batch_size, 1)

for i in range(iters_num):
    batch_mask = np.random.choice(train_size, batch_size)
    x_batch = x_train[batch_mask]
    t_batch = t_train[batch_mask]

    grad = network.gradient(x_batch,t_batch)

    for key in ('W1', 'b1', 'W2', 'b2'):
        network.params[key] -= learning_rate * grad[key]

    loss = network.loss(x_batch,t_batch)
    train_loss_list.append(loss)
# i = 9999
    if i % iter_per_epoch == 0:
        train_acc = network.accuracy(x_train,t_train)
        test_acc = network.accuracy(x_test,t_test)
        train_acc_list.append(train_acc)
        test_acc_list.append(test_acc)
        print("train acc, test acc | "+ str(train_acc) + "," + str(test_acc))

import numpy as np

from p88func import *

import sys,os

sys.path.append(os.pardir)

from mnist import load_mnist

(x_train, t_train),(x_test,t_test) = load_mnist(normalize= True, one_hot_label=True)

network = TwoLayerNet(input_size = 784, hidden_size = 50, output_size = 10)

iters_num = 10000

train_size = x_train.shape[0] #60,000

batch_size = 100

learning_rate = 0.1

train_loss_list = []

train_acc_list = []

test_acc_list = []

iter_per_epoch = max(train_size / batch_size, 1)

for i in range(iters_num):

batch_mask = np.random.choice(train_size, batch_size)

x_batch = x_train[batch_mask]

t_batch = t_train[batch_mask]

grad = network.gradient(x_batch,t_batch)

for key in ('W1', 'b1', 'W2', 'b2'):

network.params[key] -= learning_rate * grad[key]

loss = network.loss(x_batch,t_batch)

train_loss_list.append(loss)

# i = 9999

if i % iter_per_epoch == 0:

train_acc = network.accuracy(x_train,t_train)

test_acc = network.accuracy(x_test,t_test)

train_acc_list.append(train_acc)

test_acc_list.append(test_acc)

print("train acc, test acc | "+ str(train_acc) + "," + str(test_acc))

epoch 단위로 평가하는 코드이다. epoch 이란? 하나의 단위로 1 epoch은 학습에서 훈련 데이터를 모두 소진했을 때의 횟수에 해당, 예로 훈련 데이터 10,000개를 100개의 미니배치로…

머신러닝 - Deep Learning from scratch2019년 3월 17일

M.L (p.141)

import numpy as np
import sys,os
sys.path.append(os.pardir)
from p88func import *
from mnist import load_mnist

(x_train, t_train),(x_test,t_test) = load_mnist(normalize= True, one_hot_label=True)

train_loss_list = []

iters_num = 10000
train_size = x_train.shape[0]
batch_size = 100
learning_rate = 0.1
network = TwoLayerNet(input_size = 784, hidden_size = 50, output_size=10)

for i in range(iters_num):
    batch_mask = np.random.choice(train_size,batch_size)
    x_batch = x_train[batch_mask]
    t_batch = t_train[batch_mask]

    grad = network.gradient(x_batch,t_batch)

    for key in ('W1', 'b1', 'W2', 'b2'):
        network.params[key] -= learning_rate * grad[key]

    loss = network.loss(x_batch,t_batch)
    train_loss_list.append(loss)

import numpy as np

import sys,os

sys.path.append(os.pardir)

from p88func import *

from mnist import load_mnist

(x_train, t_train),(x_test,t_test) = load_mnist(normalize= True, one_hot_label=True)

train_loss_list = []

iters_num = 10000

train_size = x_train.shape[0]

batch_size = 100

learning_rate = 0.1

network = TwoLayerNet(input_size = 784, hidden_size = 50, output_size=10)

for i in range(iters_num):

batch_mask = np.random.choice(train_size,batch_size)

x_batch = x_train[batch_mask]

t_batch = t_train[batch_mask]

grad = network.gradient(x_batch,t_batch)

for key in ('W1', 'b1', 'W2', 'b2'):

network.params[key] -= learning_rate * grad[key]

loss = network.loss(x_batch,t_batch)

train_loss_list.append(loss)

MNIST 데이터를 가지고 미니배치 학습을 구현한 코드이다. 훈련데이터와 테스트 데이터로 나누었고 정규화와 one_hot_encoding을 하였다. 앞서 만들었던 TwoLayerNet(신경망) 을 사용하여 학습한다. iters_num= 10,000이므로 for문을 10,000번…

머신러닝 - Deep Learning from scratch2019년 3월 17일

M.L (p.137)

import numpy as np
from p88func import *
import sys,os
sys.path.append(os.pardir)
from mnist import load_mnist

class TwoLayerNet:
    def __init__(self, input_size, hidden_size, output_size, weight_init_std=0.01):
        self.params = {}
        self.params['W1'] = weight_init_std * np.random.randn(input_size,hidden_size)
        self.params['b1'] = np.zeros(hidden_size)
        self.params['W2'] = weight_init_std * np.random.randn(hidden_size,output_size)
        self.params['b2'] = np.zeros(output_size)

    def predict(self,x):
        W1, W2 = self.params['W1'], self.params['W2']
        b1, b2 = self.params['b1'], self.params['b2']

        a1 = np.dot(x,W1) + b1
        z1 = sigmoid(a1)
        a2 = np.dot(z1,W2) + b2
        y = softmax(a2)

        return y

    def loss(self,x,t):
        y = self.predict(x)

        return cross_entropy_error(y,t)

    def accuracy(self,x,t):
        y = self.predict(x)
        y = np.argmax(y,axis=1)
        t = np.argmax(t,axis=1)

        accuracy = np.sum(y == t) / float(x.shape[0])
        return accuracy

    def numerical_gradient(self,x,t):
        loss_W = lambda W: self.loss(x,t)

        grads = {}
        grads['W1'] = numerical_gradient(loss_W, self.params['W1'])
        grads['b1'] = numerical_gradient(loss_W, self.params['b1'])
        grads['W2'] = numerical_gradient(loss_W, self.params['W2'])
        grads['b2'] = numerical_gradient(loss_W, self.params['b2'])

        return grads

net = TwoLayerNet(input_size=784, hidden_size=100, output_size=10)
print(net.params['W1'].shape)
print(net.params['b1'].shape)
print(net.params['W2'].shape)
print(net.params['b2'].shape)

x = np.random.randn(100,784)
y = net.predict(x)

x = np.random.randn(100,784)
t = np.random.randn(100,10)

grads = net.numerical_gradient(x,t)
print(grads['W1'].shape)
print(grads['b1'].shape)
print(grads['W2'].shape)
print(grads['b2'].shape)

import numpy as np

from p88func import *

import sys,os

sys.path.append(os.pardir)

from mnist import load_mnist

class TwoLayerNet:

def __init__(self, input_size, hidden_size, output_size, weight_init_std=0.01):

self.params = {}

self.params['W1'] = weight_init_std * np.random.randn(input_size,hidden_size)

self.params['b1'] = np.zeros(hidden_size)

self.params['W2'] = weight_init_std * np.random.randn(hidden_size,output_size)

self.params['b2'] = np.zeros(output_size)

def predict(self,x):

W1, W2 = self.params['W1'], self.params['W2']

b1, b2 = self.params['b1'], self.params['b2']

a1 = np.dot(x,W1) + b1

z1 = sigmoid(a1)

a2 = np.dot(z1,W2) + b2

y = softmax(a2)

return y

def loss(self,x,t):

y = self.predict(x)

return cross_entropy_error(y,t)

def accuracy(self,x,t):

y = self.predict(x)

y = np.argmax(y,axis=1)

t = np.argmax(t,axis=1)

accuracy = np.sum(y == t) / float(x.shape[0])

return accuracy

def numerical_gradient(self,x,t):

loss_W = lambda W: self.loss(x,t)

grads = {}

grads['W1'] = numerical_gradient(loss_W, self.params['W1'])

grads['b1'] = numerical_gradient(loss_W, self.params['b1'])

grads['W2'] = numerical_gradient(loss_W, self.params['W2'])

grads['b2'] = numerical_gradient(loss_W, self.params['b2'])

return grads

net = TwoLayerNet(input_size=784, hidden_size=100, output_size=10)

print(net.params['W1'].shape)

print(net.params['b1'].shape)

print(net.params['W2'].shape)

print(net.params['b2'].shape)

x = np.random.randn(100,784)

y = net.predict(x)

x = np.random.randn(100,784)

t = np.random.randn(100,10)

grads = net.numerical_gradient(x,t)

print(grads['W1'].shape)

print(grads['b1'].shape)

print(grads['W2'].shape)

print(grads['b2'].shape)

2층 신경망을 하나의 클래스로 구현한 것이다. class name은 TwoLayerNet으로 구성요소를 하나씩 알아보자. def __init__함수로 input_size, hidden_size, output_size, weight_init_std=0.01로 초기화한다. self.params로 W1,b1,W2,b2 (weight,bias) 을 입력값에…