머신러닝 - Deep Learning from scratch2019년 3월 20일

M.L (p.161)

import numpy as np

class MulLayer:
    def __init__(self):
        self.x = None
        self.y = None

    def forward(self,x,y):
        self.x = x
        self.y = y
        out = x * y

        return out

    def backward(self,dout):
        dx = dout * self.y
        dy = dout * self.x
        return dx,dy

class AddLayer:
    def __init__(self):
        pass

    def forward(self,x,y):
        out = x + y
        return out

    def backward(self,dout):
        dx = dout
        dy = dout
        return dx,dy

import numpy as np

class MulLayer:

def __init__(self):

self.x = None

self.y = None

def forward(self,x,y):

self.x = x

self.y = y

out = x * y

return out

def backward(self,dout):

dx = dout * self.y

dy = dout * self.x

return dx,dy

class AddLayer:

def __init__(self):

pass

def forward(self,x,y):

out = x + y

return out

def backward(self,dout):

dx = dout

dy = dout

return dx,dy

backpropagation(역전파)에 덧셈과 곱셈 노드에 대한 코드이다. 역전파는 계산그래프를 거꾸로 돌아가며 각 weight와 parameter가 Y(output value)에 얼마나 영향을 미치는지(미분을 통해) chain rule 이라는 성질을 이용해…

머신러닝 - Deep Learning from scratch2019년 3월 17일

M.L (p.143)

import numpy as np
from p88func import *
import sys,os
sys.path.append(os.pardir)
from mnist import load_mnist

(x_train, t_train),(x_test,t_test) = load_mnist(normalize= True, one_hot_label=True)

network = TwoLayerNet(input_size = 784, hidden_size = 50, output_size = 10)

iters_num = 10000
train_size = x_train.shape[0] #60,000
batch_size = 100
learning_rate = 0.1

train_loss_list = []
train_acc_list = []
test_acc_list = []

iter_per_epoch = max(train_size / batch_size, 1)

for i in range(iters_num):
    batch_mask = np.random.choice(train_size, batch_size)
    x_batch = x_train[batch_mask]
    t_batch = t_train[batch_mask]

    grad = network.gradient(x_batch,t_batch)

    for key in ('W1', 'b1', 'W2', 'b2'):
        network.params[key] -= learning_rate * grad[key]

    loss = network.loss(x_batch,t_batch)
    train_loss_list.append(loss)
# i = 9999
    if i % iter_per_epoch == 0:
        train_acc = network.accuracy(x_train,t_train)
        test_acc = network.accuracy(x_test,t_test)
        train_acc_list.append(train_acc)
        test_acc_list.append(test_acc)
        print("train acc, test acc | "+ str(train_acc) + "," + str(test_acc))

import numpy as np

from p88func import *

import sys,os

sys.path.append(os.pardir)

from mnist import load_mnist

(x_train, t_train),(x_test,t_test) = load_mnist(normalize= True, one_hot_label=True)

network = TwoLayerNet(input_size = 784, hidden_size = 50, output_size = 10)

iters_num = 10000

train_size = x_train.shape[0] #60,000

batch_size = 100

learning_rate = 0.1

train_loss_list = []

train_acc_list = []

test_acc_list = []

iter_per_epoch = max(train_size / batch_size, 1)

for i in range(iters_num):

batch_mask = np.random.choice(train_size, batch_size)

x_batch = x_train[batch_mask]

t_batch = t_train[batch_mask]

grad = network.gradient(x_batch,t_batch)

for key in ('W1', 'b1', 'W2', 'b2'):

network.params[key] -= learning_rate * grad[key]

loss = network.loss(x_batch,t_batch)

train_loss_list.append(loss)

# i = 9999

if i % iter_per_epoch == 0:

train_acc = network.accuracy(x_train,t_train)

test_acc = network.accuracy(x_test,t_test)

train_acc_list.append(train_acc)

test_acc_list.append(test_acc)

print("train acc, test acc | "+ str(train_acc) + "," + str(test_acc))

epoch 단위로 평가하는 코드이다. epoch 이란? 하나의 단위로 1 epoch은 학습에서 훈련 데이터를 모두 소진했을 때의 횟수에 해당, 예로 훈련 데이터 10,000개를 100개의 미니배치로…

머신러닝 - Deep Learning from scratch2019년 3월 17일

M.L (p.141)

import numpy as np
import sys,os
sys.path.append(os.pardir)
from p88func import *
from mnist import load_mnist

(x_train, t_train),(x_test,t_test) = load_mnist(normalize= True, one_hot_label=True)

train_loss_list = []

iters_num = 10000
train_size = x_train.shape[0]
batch_size = 100
learning_rate = 0.1
network = TwoLayerNet(input_size = 784, hidden_size = 50, output_size=10)

for i in range(iters_num):
    batch_mask = np.random.choice(train_size,batch_size)
    x_batch = x_train[batch_mask]
    t_batch = t_train[batch_mask]

    grad = network.gradient(x_batch,t_batch)

    for key in ('W1', 'b1', 'W2', 'b2'):
        network.params[key] -= learning_rate * grad[key]

    loss = network.loss(x_batch,t_batch)
    train_loss_list.append(loss)

import numpy as np

import sys,os

sys.path.append(os.pardir)

from p88func import *

from mnist import load_mnist

(x_train, t_train),(x_test,t_test) = load_mnist(normalize= True, one_hot_label=True)

train_loss_list = []

iters_num = 10000

train_size = x_train.shape[0]

batch_size = 100

learning_rate = 0.1

network = TwoLayerNet(input_size = 784, hidden_size = 50, output_size=10)

for i in range(iters_num):

batch_mask = np.random.choice(train_size,batch_size)

x_batch = x_train[batch_mask]

t_batch = t_train[batch_mask]

grad = network.gradient(x_batch,t_batch)

for key in ('W1', 'b1', 'W2', 'b2'):

network.params[key] -= learning_rate * grad[key]

loss = network.loss(x_batch,t_batch)

train_loss_list.append(loss)

MNIST 데이터를 가지고 미니배치 학습을 구현한 코드이다. 훈련데이터와 테스트 데이터로 나누었고 정규화와 one_hot_encoding을 하였다. 앞서 만들었던 TwoLayerNet(신경망) 을 사용하여 학습한다. iters_num= 10,000이므로 for문을 10,000번…

머신러닝 - Deep Learning from scratch2019년 3월 17일

M.L (p.137)

import numpy as np
from p88func import *
import sys,os
sys.path.append(os.pardir)
from mnist import load_mnist

class TwoLayerNet:
    def __init__(self, input_size, hidden_size, output_size, weight_init_std=0.01):
        self.params = {}
        self.params['W1'] = weight_init_std * np.random.randn(input_size,hidden_size)
        self.params['b1'] = np.zeros(hidden_size)
        self.params['W2'] = weight_init_std * np.random.randn(hidden_size,output_size)
        self.params['b2'] = np.zeros(output_size)

    def predict(self,x):
        W1, W2 = self.params['W1'], self.params['W2']
        b1, b2 = self.params['b1'], self.params['b2']

        a1 = np.dot(x,W1) + b1
        z1 = sigmoid(a1)
        a2 = np.dot(z1,W2) + b2
        y = softmax(a2)

        return y

    def loss(self,x,t):
        y = self.predict(x)

        return cross_entropy_error(y,t)

    def accuracy(self,x,t):
        y = self.predict(x)
        y = np.argmax(y,axis=1)
        t = np.argmax(t,axis=1)

        accuracy = np.sum(y == t) / float(x.shape[0])
        return accuracy

    def numerical_gradient(self,x,t):
        loss_W = lambda W: self.loss(x,t)

        grads = {}
        grads['W1'] = numerical_gradient(loss_W, self.params['W1'])
        grads['b1'] = numerical_gradient(loss_W, self.params['b1'])
        grads['W2'] = numerical_gradient(loss_W, self.params['W2'])
        grads['b2'] = numerical_gradient(loss_W, self.params['b2'])

        return grads

net = TwoLayerNet(input_size=784, hidden_size=100, output_size=10)
print(net.params['W1'].shape)
print(net.params['b1'].shape)
print(net.params['W2'].shape)
print(net.params['b2'].shape)

x = np.random.randn(100,784)
y = net.predict(x)

x = np.random.randn(100,784)
t = np.random.randn(100,10)

grads = net.numerical_gradient(x,t)
print(grads['W1'].shape)
print(grads['b1'].shape)
print(grads['W2'].shape)
print(grads['b2'].shape)

import numpy as np

from p88func import *

import sys,os

sys.path.append(os.pardir)

from mnist import load_mnist

class TwoLayerNet:

def __init__(self, input_size, hidden_size, output_size, weight_init_std=0.01):

self.params = {}

self.params['W1'] = weight_init_std * np.random.randn(input_size,hidden_size)

self.params['b1'] = np.zeros(hidden_size)

self.params['W2'] = weight_init_std * np.random.randn(hidden_size,output_size)

self.params['b2'] = np.zeros(output_size)

def predict(self,x):

W1, W2 = self.params['W1'], self.params['W2']

b1, b2 = self.params['b1'], self.params['b2']

a1 = np.dot(x,W1) + b1

z1 = sigmoid(a1)

a2 = np.dot(z1,W2) + b2

y = softmax(a2)

return y

def loss(self,x,t):

y = self.predict(x)

return cross_entropy_error(y,t)

def accuracy(self,x,t):

y = self.predict(x)

y = np.argmax(y,axis=1)

t = np.argmax(t,axis=1)

accuracy = np.sum(y == t) / float(x.shape[0])

return accuracy

def numerical_gradient(self,x,t):

loss_W = lambda W: self.loss(x,t)

grads = {}

grads['W1'] = numerical_gradient(loss_W, self.params['W1'])

grads['b1'] = numerical_gradient(loss_W, self.params['b1'])

grads['W2'] = numerical_gradient(loss_W, self.params['W2'])

grads['b2'] = numerical_gradient(loss_W, self.params['b2'])

return grads

net = TwoLayerNet(input_size=784, hidden_size=100, output_size=10)

print(net.params['W1'].shape)

print(net.params['b1'].shape)

print(net.params['W2'].shape)

print(net.params['b2'].shape)

x = np.random.randn(100,784)

y = net.predict(x)

x = np.random.randn(100,784)

t = np.random.randn(100,10)

grads = net.numerical_gradient(x,t)

print(grads['W1'].shape)

print(grads['b1'].shape)

print(grads['W2'].shape)

print(grads['b2'].shape)

2층 신경망을 하나의 클래스로 구현한 것이다. class name은 TwoLayerNet으로 구성요소를 하나씩 알아보자. def __init__함수로 input_size, hidden_size, output_size, weight_init_std=0.01로 초기화한다. self.params로 W1,b1,W2,b2 (weight,bias) 을 입력값에…

머신러닝 - Deep Learning from scratch2019년 3월 17일

M.L (p.134)

import numpy as np
from p88func import *

class simpleNet:
    def __init__(self):
        self.W = np.random.randn(2,3)

    def predict(self,x):
        return np.dot(x,self.W)

    def loss(self,x,t):
        z = self.predict(x)
        y = softmax(z)
        loss = cross_entropy_error(y,t)

        return loss
net = simpleNet()
print(net.W)
x = np.array([0.6,0.9])
p = net.predict(x)
print(p)
print(np.argmax(p))
t= np.array([0,0,1])
d = net.loss(x,t)
print(d)

def f(W):
    return net.loss(x,t)

dW = numerical_gradient(f,net.W)
print(dW)

import numpy as np

from p88func import *

class simpleNet:

def __init__(self):

self.W = np.random.randn(2,3)

def predict(self,x):

return np.dot(x,self.W)

def loss(self,x,t):

z = self.predict(x)

y = softmax(z)

loss = cross_entropy_error(y,t)

return loss

net = simpleNet()

print(net.W)

x = np.array([0.6,0.9])

p = net.predict(x)

print(p)

print(np.argmax(p))

t= np.array([0,0,1])

d = net.loss(x,t)

print(d)

def f(W):

return net.loss(x,t)

dW = numerical_gradient(f,net.W)

print(dW)

(2,3) 같은 다차원배열일 경우의 기울기를 구한다. simpleNet은 입력 값 x와 normal distribution로 초기화한 W 값(가중치) 를 행렬곱하여 (xW) 그 값을 softmax 값으로 변환하고 정답레이블(t)와…

머신러닝 - Deep Learning from scratch2019년 3월 17일

M.L (p.131)

import numpy as np
from p88func import *

def gradient_descent(f, init_x, lr=0.01, step_num=100):
    x = init_x

    for i in range(step_num):
        grad = numerical_gradient(f,x)
        x -= lr * grad

    return x

def function_2(x):
    return x[0]**2 + x[1]**2

init_x = np.array([-3.0,4.0])
d = gradient_descent(function_2,init_x=init_x,lr=0.1,step_num=100)
print(d)

init_x = np.array([-3.0,4.0])
print(gradient_descent(function_2,init_x=init_x, lr = 10.0, step_num=100))
init_x = np.array([-3.0,4.0])
print(gradient_descent(function_2,init_x=init_x, lr = 1e-10, step_num=100))

import numpy as np

from p88func import *

def gradient_descent(f, init_x, lr=0.01, step_num=100):

x = init_x

for i in range(step_num):

grad = numerical_gradient(f,x)

x -= lr * grad

return x

def function_2(x):

return x[0]**2 + x[1]**2

init_x = np.array([-3.0,4.0])

d = gradient_descent(function_2,init_x=init_x,lr=0.1,step_num=100)

print(d)

init_x = np.array([-3.0,4.0])

print(gradient_descent(function_2,init_x=init_x, lr = 10.0, step_num=100))

init_x = np.array([-3.0,4.0])

print(gradient_descent(function_2,init_x=init_x, lr = 1e-10, step_num=100))

gradient_descent method 로 경사법 수식 , (eta) 는 갱신하는 양을 나타내고 신경망에서는 이를 학습률(learning rate)라 부른다. 이 parameter는 사용자가 정의하며 learning rate * 편미분…

머신러닝 - Deep Learning from scratch2019년 3월 16일

M.L (p.127)

import numpy as np

def function_2(x):
    return x[0] ** 2 + x[1] ** 2
    # return np.sum(x**2)

def numerical_gradient(f,x):
    h = 1e-4
    grad = np.zeros_like(x)

    for idx in range(x.size):
        tmp_val = x[idx]
        x[idx] = tmp_val + h
        fxh1 = f(x)

        x[idx] = tmp_val - h
        fxh2 = f(x)

        grad[idx] = (fxh1 - fxh2) / (2*h)
        x[idx] = tmp_val
    return grad

print(numerical_gradient(function_2, np.array([3.0,4.0])))
print(numerical_gradient(function_2, np.array([0.0,2.0])))

import numpy as np

def function_2(x):

return x[0] ** 2 + x[1] ** 2

# return np.sum(x**2)

def numerical_gradient(f,x):

h = 1e-4

grad = np.zeros_like(x)

for idx in range(x.size):

tmp_val = x[idx]

x[idx] = tmp_val + h

fxh1 = f(x)

x[idx] = tmp_val - h

fxh2 = f(x)

grad[idx] = (fxh1 - fxh2) / (2*h)

x[idx] = tmp_val

return grad

print(numerical_gradient(function_2, np.array([3.0,4.0])))

print(numerical_gradient(function_2, np.array([0.0,2.0])))

편미분을 간단히 하나의 함수로 나타내기 위한 코드 function_2 = x[0]^2 + x[1]^2 식을 가지고 있고, numerical_gradient 를 보면 for문을 이용해 각 인덱스의 중앙차분을 구하여…

머신러닝 - Deep Learning from scratch2019년 3월 16일

M.L (p.125)

import numpy as np

def numerical_diff(f,x):
    h = 1e-4
    return (f(x+h) - f(x-h)) / (2*h)

def function_2(x):
    return x[0] ** 2 + x[1] ** 2
    # return np.sum(x**2)


def function_tmp1(x0):
    return x0 * x0 + 4.0 ** 2.0

A = numerical_diff(function_tmp1, 3.0)
print(A)

def function_tmp2(x1):
    return 3.0 ** 2.0 + x1 * x1

B = numerical_diff(function_tmp2,4.0)
print(B)

import numpy as np

def numerical_diff(f,x):

h = 1e-4

return (f(x+h) - f(x-h)) / (2*h)

def function_2(x):

return x[0] ** 2 + x[1] ** 2

# return np.sum(x**2)

def function_tmp1(x0):

return x0 * x0 + 4.0 ** 2.0

A = numerical_diff(function_tmp1, 3.0)

print(A)

def function_tmp2(x1):

return 3.0 ** 2.0 + x1 * x1

B = numerical_diff(function_tmp2,4.0)

print(B)

편미분 (둘 이상의 변수) 식 f(x0,x1) = x0^2 + x1^2 같은 간단한 식 구현 편미분을 구할 때는 구하려는 변수 외의 다른 변수 값은 상수로…

머신러닝 - Deep Learning from scratch2019년 3월 16일

M.L (p.121)

import numpy as np
import matplotlib.pyplot as plt

def numerical_diff(f,x):
    h = 1e-4
    return (f(x+h) - f(x-h)) / (2*h)

def function_1(x):
    return 0.01*x**2 + 0.1*x

x = np.arange(0.0, 20.0, 0.1)
y = function_1(x)
plt.xlabel("x")
plt.ylabel("f(x)")
plt.plot(x,y)
plt.show()

print(numerical_diff(function_1,5))
print(numerical_diff(function_1,10))

import numpy as np

import matplotlib.pyplot as plt

def numerical_diff(f,x):

h = 1e-4

return (f(x+h) - f(x-h)) / (2*h)

def function_1(x):

return 0.01*x**2 + 0.1*x

x = np.arange(0.0, 20.0, 0.1)

y = function_1(x)

plt.xlabel("x")

plt.ylabel("f(x)")

plt.plot(x,y)

plt.show()

print(numerical_diff(function_1,5))

print(numerical_diff(function_1,10))

경사법에서는 기울기 값을 기준으로 나아갈 방향을 정하는데 기울기 값을 미분으로 구할 수 있다. (기울기 값이 0에 가까워지는 지점이 최적 값을 나타내기때문에 ) 하지만 컴퓨터…

머신러닝 - Deep Learning from scratch2019년 3월 15일

M.L (p.116)

import sys,os
import numpy as np
sys.path.append(os.pardir)
from mnist import load_mnist

(x_train,t_train),(x_test,t_test) = load_mnist(normalize=True, one_hot_label=True)

train_size = x_train.shape[0]
batch_size = 10
a = batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
print(a)

import sys,os

import numpy as np

sys.path.append(os.pardir)

from mnist import load_mnist

(x_train,t_train),(x_test,t_test) = load_mnist(normalize=True, one_hot_label=True)

train_size = x_train.shape[0]

batch_size = 10

a = batch_mask = np.random.choice(train_size, batch_size)

x_batch = x_train[batch_mask]

t_batch = t_train[batch_mask]

print(a)

mini batch : 거대한 데이터셋을 다 계산해보는 건 시간적, 비용적으로 부담이 크기 때문에 적절히 어느정도 양의 데이터를 골라 그 값을 이용해 근사치를 구한다. 예를…