새싹 AI데이터엔지니어 핀테커스 12주차 (월) - Pytorch Tutorial2 (Moon & Make Blobs XOR Dataset)

2023-11-20 53rd Class

2 Layers MLP (with Moon Dataset from Scikit-Learn)

#️⃣ 2레이어 MLP 모델 학습 및 시각화 - 비선형

code

from sklearn.datasets import make_moons  
import matplotlib.pyplot as plt  
  
import torch  
from torch.utils.data import TensorDataset  
from torch.utils.data import DataLoader  
import torch.nn as nn  
from torch.optim import SGD  
  
  
class MLP(nn.Module):  
    def __init__(self, n_features):  
        super(MLP, self).__init__()  
        self.fc1 = nn.Linear(in_features=n_features, out_features=3)  
        self.fc1_act = nn.Sigmoid()  
        self.fc2 = nn.Linear(in_features=3, out_features=1)  
        self.fc2_act = nn.Sigmoid()  
  
    def forward(self, x):  
        x = self.fc1(x)  
        x = self.fc1_act(x)  
        x = self.fc2(x)  
        x = self.fc2_act(x)  
        x = x.view(-1)  
        return x  
  
  
def get_dataset(N_SAMPLES=300, BATCH_SIZE=8):  
    X, y = make_moons(n_samples=N_SAMPLES, noise=0.2)  
    dataset = TensorDataset(torch.FloatTensor(X), torch.FloatTensor(y))  
    dataloader = DataLoader(dataset, batch_size=BATCH_SIZE)  
    return dataloader  
  
  
def get_device():  
    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  
    print(f"curr device = {DEVICE}")  
    return DEVICE  
  
  
def train(dataloader, N_SAMPLES, model, loss_function, optimizer, DEVICE):    
    epoch_loss, n_corrects = 0., 0  
    for X, y in dataloader:  
        X, y = X.to(DEVICE), y.to(DEVICE)  
  
        pred = model(X)  
        loss = loss_function(pred, y)  
  
        optimizer.zero_grad()  
        loss.backward()  
        optimizer.step()  
  
        epoch_loss += loss.item() * len(X)  
        pred = (pred > 0.5).type(torch.float)  
        n_corrects += (pred == y).sum().item()  
  
    epoch_loss /= N_SAMPLES  
    epoch_accr = n_corrects / N_SAMPLES  
  
    print(f"EPOCH: {epoch + 1}", end="\t")  
    print(f"Accuracy: {epoch_accr}", end="\t")  
    print(f"Loss: {epoch_loss}")  
  
    return epoch_loss, epoch_accr  
  
  
def vis_lossees_accs(losses, accs):  
    fig, axes = plt.subplots(2, 1, figsize=(7, 3))  
    axes[0].plot(losses)  
    axes[1].plot(accs)  
  
    axes[1].set_xlabel("Epoch", fontsize=15)  
    axes[0].set_ylabel("Loss", fontsize=15)  
    axes[1].set_ylabel("Accuracy", fontsize=15)  
    axes[0].tick_params(labelsize=10)  
    axes[1].tick_params(labelsize=10)  
    fig.suptitle("2-layer Model Eval Metrics by Epoch", fontsize=16)  
    fig.tight_layout()  
    plt.show()  
  
  
N_SAMPLES = 300  
BATCH_SIZE = 8  
EPOCHS = 100  
LR = 0.01  
n_features = 2  
DEVICE = get_device()  
  
dataloader = get_dataset(N_SAMPLES, BATCH_SIZE)  
  
model = MLP(n_features=n_features).to(DEVICE)  
loss_function = nn.BCELoss()  
optimizer = SGD(model.parameters(), lr=LR)  
  
losses, accs = list(), list()  
  
for epoch in range(EPOCHS):  
    epoch_loss, epoch_acc = train(dataloader, N_SAMPLES, model,  
                                  loss_function, optimizer, DEVICE)  
  
    losses.append(epoch_loss)  
    accs.append(epoch_acc)  
  
vis_lossees_accs(losses, accs)

curr device = cuda
EPOCH: 1	Accuracy: 0.5	Loss: 0.7462213285764059
EPOCH: 2	Accuracy: 0.5	Loss: 0.7187317283948262
EPOCH: 3	Accuracy: 0.5	Loss: 0.6986713870366414
EPOCH: 4	Accuracy: 0.5	Loss: 0.683925724029541
EPOCH: 5	Accuracy: 0.5	Loss: 0.6729125261306763
EPOCH: 6	Accuracy: 0.5	Loss: 0.6644860506057739
EPOCH: 7	Accuracy: 0.54	Loss: 0.6578351354598999
EPOCH: 8	Accuracy: 0.6733333333333333	Loss: 0.6523939347267151
EPOCH: 9	Accuracy: 0.7233333333333334	Loss: 0.6477716318766276
EPOCH: 10	Accuracy: 0.7533333333333333	Loss: 0.6437000711758931
...
EPOCH: 90	Accuracy: 0.8133333333333334	Loss: 0.4312522117296855
EPOCH: 91	Accuracy: 0.8133333333333334	Loss: 0.42970679759979247
EPOCH: 92	Accuracy: 0.8133333333333334	Loss: 0.4281863621870677
EPOCH: 93	Accuracy: 0.8133333333333334	Loss: 0.4266904242833455
EPOCH: 94	Accuracy: 0.8133333333333334	Loss: 0.4252185300985972
EPOCH: 95	Accuracy: 0.8133333333333334	Loss: 0.4237701960404714
EPOCH: 96	Accuracy: 0.8133333333333334	Loss: 0.42234497507413227
EPOCH: 97	Accuracy: 0.8133333333333334	Loss: 0.42094242175420127
EPOCH: 98	Accuracy: 0.8133333333333334	Loss: 0.4195620401700338
EPOCH: 99	Accuracy: 0.8133333333333334	Loss: 0.4182034146785736
EPOCH: 100	Accuracy: 0.8133333333333334	Loss: 0.41686609824498494

---

2 Layers MLP (with 4 Centroids Dataset from Scikit-Learn)

#️⃣ 2레이어 MLP 모델 학습 및 시각화 - XOR

dl_13_pytorch_basics2.py

from sklearn.datasets import make_moons  
import matplotlib.pyplot as plt  
  
import torch  
from torch.utils.data import TensorDataset  
from torch.utils.data import DataLoader  
import torch.nn as nn  
from torch.optim import SGD  
  
  
class MLP(nn.Module):  
    def __init__(self, n_features, n_hdn_nrns):  
        super(MLP, self).__init__()  
        self.fc1 = nn.Linear(in_features=n_features, out_features=n_hdn_nrns)  
        self.fc1_act = nn.Sigmoid()  
        self.fc2 = nn.Linear(in_features=n_hdn_nrns, out_features=1)  
        self.fc2_act = nn.Sigmoid()  
  
    def forward(self, x):  
        x = self.fc1(x)  
        x = self.fc1_act(x)  
        x = self.fc2(x)  
        x = self.fc2_act(x)  
        x = x.view(-1)  
        return x   


def get_device():  
    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  
    print(f"curr device = {DEVICE}")  
    return DEVICE  
  
  
def train(dataloader, N_SAMPLES, model, loss_function, optimizer, DEVICE):  
    epoch_loss, n_corrects = 0., 0  
    for X, y in dataloader:  
        X, y = X.to(DEVICE), y.to(DEVICE)  
  
        pred = model(X)  
        loss = loss_function(pred, y)  
  
        optimizer.zero_grad()  
        loss.backward()  
        optimizer.step()  
  
        epoch_loss += loss.item() * len(X)  
        pred = (pred > 0.5).type(torch.float)  
        n_corrects += (pred == y).sum().item()  
  
    epoch_loss /= N_SAMPLES  
    epoch_accr = n_corrects / N_SAMPLES  
  
    return epoch_loss, epoch_accr  
  
  
def vis_losses_accs(losses, accs):  
    fig, axes = plt.subplots(2, 1, figsize=(14, 5))  
    axes[0].plot(losses)  
    axes[1].plot(accs)  
  
    axes[1].set_xlabel("Epoch", fontsize=15)  
    axes[0].set_ylabel("Loss", fontsize=15)  
    axes[1].set_ylabel("Accuracy", fontsize=15)  
    axes[0].tick_params(labelsize=10)  
    axes[1].tick_params(labelsize=10)  
    fig.suptitle("2-layer Model Eval Metrics by Epoch", fontsize=16)   

dl_14_pytorch_basics3.py

import numpy as np  
import torch  
from sklearn.datasets import make_blobs  
  
from dl_13_pytorch_basics2 import get_device, train, vis_losses_accs, MLP  
import torch.nn as nn  
from torch.optim import SGD  
from torch.utils.data import TensorDataset  
from torch.utils.data import DataLoader  
import matplotlib.pyplot as plt  
from dataclasses import dataclass  
  
  
@dataclass  
class Constants:  
    N_SAMPLES: int  
    BATCH_SIZE: int  
    EPOCHS: int  
    LR: float  
    n_features: int  
    DEVICE: torch.device  
    PATH: str  
    SEED: int  
  
def get_grid_data(xlim, ylim):  
    x1 = np.linspace(xlim[0], xlim[1], 100)  
    x2 = np.linspace(ylim[0], ylim[1], 100)  
    X1, X2 = np.meshgrid(x1, x2)  
    X_db = np.hstack([X1.reshape(-1, 1), X2.reshape(-1, 1)])  
    return X_db  
  
  
def get_dataset(N_SAMPLES=100, BATCH_SIZE=8, SEED=0):  
    HALF_N_SAMPLES = int(N_SAMPLES / 2)  
  
    # create data  
    centers1 = [(-1, -1), (-1, 1)]  
    centers2 = [(1, 1), (1, -1)]  
    X1, y1 = make_blobs(n_samples=HALF_N_SAMPLES, centers=centers1, n_features=2, cluster_std=0.3, random_state=SEED)  
    X2, y2 = make_blobs(n_samples=HALF_N_SAMPLES, centers=centers2, n_features=2, cluster_std=0.3, random_state=SEED)  
  
    X = np.vstack([X1, X2])  
    y = np.vstack([y1, y2]).flatten()  
  
    # grid  
    xylim, fig, ax = visualize_dataset(X, y)  
    X_db = get_grid_data(*xylim)  
    X_db = TensorDataset(torch.FloatTensor(X_db))  
  
    # dataloader  
    dataset = TensorDataset(torch.FloatTensor(X), torch.FloatTensor(y))  
    dataloader = DataLoader(dataset, batch_size=BATCH_SIZE)  
    return dataloader, dataset, X_db, X, y  
  
  
def visualize_dataset(X, y):  
    fig, ax = plt.subplots(figsize=(10, 10))  
    ax.scatter(X[:, 0], X[:, 1], c=y, cmap='bwr')  
    ax.tick_params(labelsize=15)  
    # plt.show()  
    xylim = ax.get_xlim(), ax.get_ylim()  
    return xylim, fig, ax  
  
  
def vis_meshgrid(X_db, pred, X, y):  
    fig, ax = plt.subplots(figsize=(10, 10))  
    ax.scatter(X[:, 0], X[:, 1], c=y, cmap='bwr')  
    ax.scatter(X_db[:, 0], X_db[:, 1], c=pred, cmap='bwr', alpha=0.1)  
    plt.show()  
  
  
def xor_with_mlp(c):  
    # Data  
    dataloader, dataset, X_db, X, y = get_dataset(N_SAMPLES=c.N_SAMPLES, BATCH_SIZE=c.BATCH_SIZE, SEED=c.SEED)  
  
    # Model  
    model = MLP(n_features=c.n_features, n_hdn_nrns=3).to(c.DEVICE)  
    loss_function = nn.BCELoss()  
    optimizer = SGD(model.parameters(), lr=c.LR)  
  
    # Training  
    losses, accs = list(), list()  
  
    for epoch in range(c.EPOCHS):  
        epoch_loss, epoch_acc = train(dataloader, c.N_SAMPLES, model,  
                                      loss_function, optimizer, c.DEVICE)  
  
        losses.append(epoch_loss)  
        accs.append(epoch_acc)  
  
        if epoch % 100 == 0:  
            print(f"EPOCH: {epoch + 1}", end="\t")  
            print(f"Accuracy: {epoch_acc}", end="\t")  
            print(f"Loss: {epoch_loss}")  
  
    # Save Model  
    torch.save(model.state_dict(), c.PATH)  
  
    # Predict  
    X_db = X_db.tensors[0].to(c.DEVICE)  
    # 1번 방법(faster) torch.no_grad() autograd engine 비활성화  
    with torch.no_grad():  
        pred = model(X_db).to("cpu")  
  
    # 2번 방법: model에서 바로 예측 후 detach()하여 모델의 예측값만 받기  
    # pred = model(X_db).to("cpu")  
    # pred = pred.to("cpu").detach()  
    X_db = X_db.to("cpu").detach().numpy()  
    pred = (pred > 0.5).type(torch.float).numpy()  
  
    # Visualization  
    vis_losses_accs(losses, accs)  
    vis_meshgrid(X_db, pred, X, y)  
  
  
def xor_with_mlp_eval(c):  
    '''torch 모델 불러와서 추론만 하기 -> 수정중'''  
    # Data  
    _, dataset, X_db, X, y = get_dataset(N_SAMPLES=c.N_SAMPLES, BATCH_SIZE=c.BATCH_SIZE, SEED=c.SEED)  
  
    # Model  
    model = MLP(n_features=c.n_features, n_hdn_nrns=2).to(c.DEVICE)  
    model.load_state_dict(torch.load(c.PATH))  
    model.eval()  
  
    # Predict  
    X_db = X_db.tensors[0].to(c.DEVICE)  
    with torch.no_grad():  
        pred = model(X_db).to("cpu")  
  
    X_db = X_db.to("cpu").detach().numpy()  
    pred = (pred > 0.5).type(torch.float).numpy()  
  
    # Visualization  
    vis_meshgrid(X_db, pred, X, y)  
  
  
  
if __name__ == '__main__':  
    constants = Constants(  
        N_SAMPLES=300,  
        BATCH_SIZE=8,  
        EPOCHS=1000,  
        LR=0.01,  
        n_features=2,  
        DEVICE=get_device(),  
        PATH="model/xor_params.pt",  
        SEED=8  
    )  
    np.random.seed(constants.SEED)  
    xor_with_mlp(constants)  
    # xor_with_mlp_eval(constants)

curr device = cuda
EPOCH: 1	Accuracy: 0.5	Loss: 0.7239237435658773
EPOCH: 101	Accuracy: 0.5	Loss: 0.6856461540857951
EPOCH: 201	Accuracy: 0.6966666666666667	Loss: 0.6457256937026977
EPOCH: 301	Accuracy: 0.8433333333333334	Loss: 0.5101923743883768
EPOCH: 401	Accuracy: 0.9766666666666667	Loss: 0.32331072449684145
EPOCH: 501	Accuracy: 0.98	Loss: 0.2138163427511851
EPOCH: 601	Accuracy: 0.9833333333333333	Loss: 0.15879764755566914
EPOCH: 701	Accuracy: 0.9833333333333333	Loss: 0.12822827080885568
EPOCH: 801	Accuracy: 0.9833333333333333	Loss: 0.10937805230418841
EPOCH: 901	Accuracy: 0.9833333333333333	Loss: 0.09680191258589427

base dataset	decision boundary