4.B.2.Create_model_and_training.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"4.B.2.Create_model_and_training.ipynb의 사본","provenance":[{"file_id":"19j0pSQt5Ry_TXerB0boEB9AG_685RvzG","timestamp":1602549540759},{"file_id":"1MaNz2WG1ldsa7OAFUYQRR1WaRKjkEpkT","timestamp":1602391981272}],"collapsed_sections":[],"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"id":"ug_Z28Sv2I9r"},"source":["# 모델 생성 및 학습\n","- 처음 시작할 때는 데이터를 5fold로 나눴기에 5개의 모델을 학습하였으나, 마지막에는 빠르게 학습되면서 성능이 가장 좋은 2개만 사용하였습니다. \n","- 처음에는 validation 셋을 검증용으로 보면서 오버피팅을 체크하였지만 마지막에는 validation 셋까지 다 넣고 학습하였습니다.\n","\n","- 이 데이터의 경우 로컬 미니마에 빠지기 쉬워 특히 학습이 어려운 편입니다.\n","- 따라서 일단 컨볼루션 오토인코더로 이미지의 특징을 추출하는 CNN을 먼저 학습하고, 이후에 학습된 CNN을 기반으로 이미지 캡셔닝 모델을 학습합니다.\n","- 컨볼루션 오토인코더는 기존 VGG, ResNet, Densenet, InceptionV3 기반의 모델을 다 테스트 해 보았는데, 모델의 크기나 퀄리티를 보아 DenseNet에서 채널 수를 조정한 모델을 기반으로 사용하였습니다.\n","\n","- 캡셔닝 모델은 베이스라인 코드를 기반으로 작성하였는데, 인코더 부분을 컨볼루션 오토인코더에서 학습된 Densenet으로, 디코더부분을 Transformer를 사용하였습니다.\n"]},{"cell_type":"code","metadata":{"id":"xKGB5-34qNaC"},"source":["from IPython.display import clear_output\n","\n","!add-apt-repository -y ppa:alessandro-strada/ppa; \n","!apt-get update;\n","!apt-get install -y google-drive-ocamlfuse;\n","clear_output()\n","\n","!mkdir google_drive;\n","!google-drive-ocamlfuse -headless -label dacon_smiles -id 406775554485-vqr231cgnpofc9mkm7sr0e3uq32emf11.apps.googleusercontent.com -secret iFy1t7pKRjOzBuWHbUB-cM8V;\n","\n","!sed -i 's/team_drive_id=0AOLSYhuNgxEsUk9PVA/team_drive_id=/' ~/.gdfuse/dacon_smiles/config\n","!sed -i 's/team_drive_id=/team_drive_id=0AOLSYhuNgxEsUk9PVA/' ~/.gdfuse/dacon_smiles/config\n","!google-drive-ocamlfuse -label dacon_smiles google_drive/\n","\n","# !fusermount -u google_drive\n","\n","# rdkit 2020.03.3 버전 다운로드\n","!pip install kora -q\n","import kora.install.rdkit\n","\n","import os\n","import os.path as pth\n","\n","### 저는 코랩에서 구글 드라이브를 네트워크 마운트해서 사용했기 때문에 경로가 이와 같이 됩니다.\n","google_drive_base_path = 'google_drive/chemical/'\n","\n","clear_output()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"j1UN3LYJzFgd"},"source":["import os\n","import os.path as pth\n","\n","data_base_path = 'data'\n","os.makedirs(data_base_path, exist_ok=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"RCVVTq4szcYU"},"source":["import tensorflow as tf\n","\n","import matplotlib.pyplot as plt\n","\n","from sklearn.model_selection import train_test_split\n","from sklearn.utils import shuffle\n","\n","import functools\n","\n","import random\n","import numpy as np\n","import pandas as pd\n","import os\n","import time\n","import cv2\n","from tqdm import tqdm\n","from glob import glob\n","\n","import kora.install.rdkit\n","\n","import rdkit\n","from rdkit import Chem\n","from rdkit import DataStructs\n","from rdkit import RDLogger\n","from rdkit.Chem import Draw\n","import multiprocessing\n","\n","RDLogger.DisableLog('rdApp.*')  \n","\n","from IPython.display import clear_output\n","from IPython.core.interactiveshell import InteractiveShell\n","InteractiveShell.ast_node_interactivity = \"all\"\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"s646shp96x4d"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"FJ47kd1F3s9f"},"source":["## 1. Transfer learning을 위한 Convolutional Autoencoder 학습하기"]},{"cell_type":"markdown","metadata":{"id":"hpY0Fqdt3_jO"},"source":["베이스라인에서는 기존 imagenet weight를 사용하였지만, 분자식 이미지는 imagenet 자연 이미지에 비해 상당히 단순하면서(라인이나 엣지 등), 자그마한 디테일 하나하나가 중요한 편입니다. (글자와 +, -와 같은 기호)  \n","\n","여러 Convolutional autoencoder를 테스트한 결과, Densenet121 기반에서 채널을 다소 낮춘 모델으로도 원래 이미지를 충분히 복원할 수 있는 것으로 확인되었습니다.\n","\n","학습은 컴페티션에서 기본으로 제공하는 데이터셋만을 이용하여 진행하였습니다."]},{"cell_type":"code","metadata":{"id":"bvmEQusd3-1s"},"source":["import tensorflow as tf\n","from tensorflow.keras.preprocessing import image\n","import cv2\n","\n","import matplotlib.pyplot as plt\n","\n","from sklearn.model_selection import train_test_split\n","from sklearn.utils import shuffle\n","\n","import numpy as np\n","import pandas as pd\n","import os\n","import os.path as pth\n","import shutil\n","import time\n","from tqdm import tqdm\n","\n","from rdkit import Chem\n","from rdkit import DataStructs\n","from rdkit import RDLogger\n","RDLogger.DisableLog('rdApp.*')  \n","\n","from IPython.display import clear_output\n","\n","from multiprocessing import Process, Queue\n","import datetime"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a9Ko5syGlCun"},"source":["import tensorflow.keras as keras\n","from keras.models import Model, Input\n","from keras.layers import Conv2D, Dense, MaxPooling2D, AveragePooling2D, GlobalAveragePooling2D\n","from keras.layers import Activation, BatchNormalization\n","from keras.layers import Concatenate\n","from keras.utils import to_categorical\n","from keras.callbacks import Callback\n","from keras.optimizers import SGD\n","import tensorflow.keras as keras\n","from keras.models import Model, Input, load_model\n","\n","import numpy as np\n","import keras.backend as K"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8IjvuZmflCyK"},"source":["BATCH_SIZE = 32\n","BUFFER_SIZE = 100\n","learning_rate = 5*1e-4\n","base_channel = 8"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ard5Rcv9lHgR"},"source":["data_base_path = 'data'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XSH9wagSlHn1"},"source":["train_path = pth.join(data_base_path, 'train')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"vR0ErWT4lHld"},"source":["with open(pth.join('data', 'train.csv'), 'r') as csv_file:\n","    data = csv_file.read()\n","    \n","all_captions = []\n","all_img_name_vector = []\n","\n","for line in data.split('\\n')[1:-1]:\n","    image_id, smiles = line.split(',')\n","    caption = '<' + smiles + '>'\n","    full_image_path = pth.join(train_path, image_id)\n","\n","    all_img_name_vector.append(full_image_path)\n","    all_captions.append(caption)\n","\n","train_captions, img_name_vector = shuffle(all_captions, all_img_name_vector, random_state=42)\n","\n","num_examples = 908765\n","train_captions = train_captions[:num_examples]\n","img_name_vector = img_name_vector[:num_examples]\n","\n","temp_img_vector = np.array(list(set(img_name_vector)))\n","img_vector_arg = np.argsort(temp_img_vector)\n","img_name_vector = temp_img_vector[img_vector_arg]\n","train_captions = np.array(train_captions)[img_vector_arg]\n","\n","captions_arg = np.argsort(train_captions)\n","img_name_vector = temp_img_vector[captions_arg]\n","train_captions = train_captions[captions_arg]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"5mqzM5CAldNG"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"22j7wc-UlgB9"},"source":["model_base_path = pth.join('model', 'checkpoint')\n","\n","model_encoder_name = 'CustomDenseNet-121'\n","model_name = 'Autoencoder_{}_trts_basech_{:03d}'.format(model_encoder_name, base_channel)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Kq5H3MpHlhQY"},"source":["img_name_train, img_name_val = train_test_split(img_name_vector, test_size=0.2, random_state=42)\n","len(img_name_train), len(img_name_val)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"xoOEuO_jljDd"},"source":["def map_func(image_path):\n","    img = tf.io.read_file(image_path)\n","    img = tf.image.decode_jpeg(img, channels=3)\n","    img = tf.dtypes.cast(img, tf.float32)\n","#     img = tf.image.resize(img, (300, 300))\n","    return img, img\n","\n","def prep_func(image):\n","    result_image = tf.keras.applications.inception_v3.preprocess_input(image)\n","    return result_image, result_image"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"suZwB6p1lxOG"},"source":["dataset_val = tf.data.Dataset.from_tensor_slices((img_name_val))\n","dataset_val = dataset_val.map(map_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","dataset_val = dataset_val.batch(BATCH_SIZE)\n","# dataset_val = dataset_val.cache()\n","# dataset_val = dataset_val.map(prep_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","dataset_val = dataset_val.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ieEpFlrFlxSv"},"source":["TEST_PATH = pth.join(data_base_path, 'test')\n","with open(pth.join(data_base_path, 'sample_submission.csv'), 'r') as csv_file:\n","    data = csv_file.read()\n","    \n","test_img_path = []\n","for line in data.split('\\n')[1:-1]:\n","    image_id, _ = line.split(',')\n","    full_image_path = pth.join(TEST_PATH, image_id)\n","    test_img_path.append(full_image_path)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"G7aKZNrzlxQy"},"source":["dataset = tf.data.Dataset.from_tensor_slices((np.concatenate([img_name_train, test_img_path])))\n","dataset = dataset.map(map_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n","# dataset = dataset.cache()\n","# dataset = dataset.map(prep_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n","\n","dataset_test = tf.data.Dataset.from_tensor_slices((test_img_path))\n","dataset_test = dataset_test.map(map_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","dataset_test = dataset_test.batch(BATCH_SIZE)\n","# dataset_test = dataset_test.cache()\n","# dataset_test = dataset_test.map(prep_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","dataset_test = dataset_test.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"l2BafchvlxKC"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"PAKy6HOCmOgg"},"source":["Convolutional autoencoder에서 사용될 모델을 정의합니다."]},{"cell_type":"code","metadata":{"id":"FqC_OUI6mNk5"},"source":["def Conv_Block(x, growth_rate, activation='relu'):\n","    x_l = BatchNormalization()(x)\n","    x_l = Activation(activation)(x_l)\n","    x_l = Conv2D(growth_rate*4, (1, 1), padding='same', kernel_initializer='he_normal')(x_l)\n","    \n","    x_l = BatchNormalization()(x_l)\n","    x_l = Activation(activation)(x_l)\n","    x_l = Conv2D(growth_rate, (3, 3), padding='same', kernel_initializer='he_normal')(x_l)\n","    \n","    x = Concatenate()([x, x_l])\n","    \n","    return x\n","\n","def Dense_Block(x, layers, growth_rate=32):\n","    for i in range(layers):\n","        x = Conv_Block(x, growth_rate)\n","    return x\n","\n","def Transition_Layer(x, compression_factor=0.5, activation='relu'):\n","    reduced_filters = int(K.int_shape(x)[-1] * compression_factor)\n","    \n","    x = BatchNormalization()(x)\n","    x = Activation(activation)(x)\n","    x = Conv2D(reduced_filters, (1, 1), padding='same', kernel_initializer='he_normal')(x)\n","    \n","    x = AveragePooling2D((2, 2), padding='same', strides=2)(x)\n","    \n","    return x\n","\n","\n","layers_in_block = {'CustomDenseNet-121' : [6, 12, 24, 16],\n","                   'CustomDenseNet-169' : [6, 12, 32, 32],\n","                   'CustomDenseNet-201' : [6, 12, 48, 32],\n","                   'CustomDenseNet-265' : [6, 12, 64, 48]}\n","\n","def DenseNet(model_input, base_growth_rate=32, densenet_type='CustomDenseNet-121'):\n","    x = Conv2D(base_growth_rate*2, (7, 7), padding='same', strides=2, kernel_initializer='he_normal')(model_input) # (224, 224, 3) -> (112, 112, 64)\n","    x = BatchNormalization()(x)\n","    x = Activation('relu')(x)\n","    \n","    x = MaxPooling2D((3, 3), padding='same', strides=2)(x) # (112, 112, 64) -> (56, 56, 64)\n","    \n","    x = Dense_Block(x, layers_in_block[densenet_type][0], base_growth_rate)\n","    x = Transition_Layer(x, compression_factor=0.5)\n","    x = Dense_Block(x, layers_in_block[densenet_type][1], base_growth_rate)\n","    x = Transition_Layer(x, compression_factor=0.5)\n","    x = Dense_Block(x, layers_in_block[densenet_type][2], base_growth_rate)\n","    x = Transition_Layer(x, compression_factor=0.5)\n","    x = Dense_Block(x, layers_in_block[densenet_type][3], base_growth_rate)\n","    \n","    model = Model(model_input, x, name=densenet_type)\n","    \n","    return model    \n","\n","\n","def build_model(base_channel=32):\n","    input_layer = tf.keras.layers.Input(shape=(300, 300, 3))\n","    encoder = DenseNet(input_layer, base_growth_rate=base_channel, densenet_type=model_encoder_name)(input_layer)\n","    \n","    x = tf.keras.layers.Conv2D(base_channel*32, (3, 3), activation='relu', padding='same')(encoder)\n","    x = tf.keras.layers.Conv2D(base_channel*32, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.Conv2D(base_channel*32, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.UpSampling2D((2, 2))(x)\n","#     x = tf.keras.layers.ZeroPadding2D((1, 1))(x)\n","    x = tf.keras.layers.Conv2D(base_channel*16, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.Conv2D(base_channel*16, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.Conv2D(base_channel*16, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.UpSampling2D((2, 2))(x)\n","#     x = tf.keras.layers.ZeroPadding2D((1, 1))(x)\n","    x = tf.keras.layers.Conv2D(base_channel*8, (3, 3), activation='relu', padding='valid')(x)\n","    x = tf.keras.layers.Conv2D(base_channel*8, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.Conv2D(base_channel*8, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.UpSampling2D((2, 2))(x)\n","#     x = tf.keras.layers.ZeroPadding2D((1, 1))(x)\n","    x = tf.keras.layers.Conv2D(base_channel*4, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.Conv2D(base_channel*4, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.UpSampling2D((2, 2))(x)\n","#     x = tf.keras.layers.ZeroPadding2D((1, 1))(x)\n","    x = tf.keras.layers.Conv2D(base_channel*2, (3, 3), activation='relu', padding='valid')(x)\n","    x = tf.keras.layers.Conv2D(base_channel*2, (3, 3), activation='relu', padding='same')(x)\n","#     x = tf.keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same')(x)\n","    x = tf.keras.layers.UpSampling2D((2, 2))(x)\n","#     x = tf.keras.layers.ZeroPadding2D((1, 1))(x)\n","    decoder = tf.keras.layers.Conv2D(3, (3, 3), activation='linear', padding='same')(x)\n","    \n","    model = tf.keras.Model(inputs=input_layer, outputs=decoder)\n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1_gUV4iBmRPv"},"source":["model = build_model(base_channel=base_channel)\n","model.summary()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nX4Ku-VimRXV"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"-LNtAIpTFWKb"},"source":["학습은 validation 셋 기준으로 early stopping을 걸었습니다."]},{"cell_type":"code","metadata":{"id":"pvXCheX0mRUp"},"source":["model_path = pth.join(model_base_path, model_name)\n","if pth.isdir(model_path):\n","    shutil.rmtree(model_path)\n","os.makedirs(model_path, exist_ok=True)\n","model_filename = pth.join(model_path, '{epoch:06d}-{loss:.6f}-{val_loss:.6f}.hdf5')\n","checkpointer = tf.keras.callbacks.ModelCheckpoint(filepath=model_filename, verbose=1, \n","                       period=1, save_best_only=True, \n","                       monitor='val_loss')\n","early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n","\n","model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(lr=learning_rate),\n","              metrics=['mse', 'mae'])\n","\n","hist = model.fit(\n","    x=dataset, epochs=10000, \n","    validation_data=dataset_val, shuffle=True,\n","    callbacks=[checkpointer, early_stopping], \n","#     batch_size=BATCH_SIZE\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"x-icBtvRH57A"},"source":["print(model_path)\n","for each_label in ['loss', 'mse', 'mae']:\n","    fig, ax = plt.subplots()\n","    ax.plot(hist.history[each_label], 'g', label='train_{}'.format(each_label))\n","    ax.plot(hist.history['val_{}'.format(each_label)], 'r', label='val_{}'.format(each_label))\n","    ax.set_xlabel('epoch')\n","    ax.set_ylabel('loss')\n","    ax.legend(loc='upper left')\n","    plt.show()\n","#     filename = 'learning_curve_{}'.format(each_label)\n","#     fig.savefig(pth.join(visualization_path, filename), transparent=True)\n","#     plt.cla()\n","#     plt.clf()\n","#     plt.close('all')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8w6l3dYlmRTW"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"_R9PCPmaFgx9"},"source":["## 2. 분자 이미지 Captioning 모델 학습하기"]},{"cell_type":"markdown","metadata":{"id":"t-xZtjXMGa2J"},"source":["모델은 이전 단계에서 학습한 DenseNet autoencoder의 encoder 부분을 캡셔닝 모델의 encoder로 사용하고, Decoder 부분을 기본 Transformer로 사용하였습니다.  \n","해당 코드의 기본 베이스는 데이콘에서 제공한 베이스라인 코드를 사용하였습니다."]},{"cell_type":"code","metadata":{"id":"WR3WR785GbFb"},"source":["import tensorflow as tf\n","from tensorflow.keras.preprocessing import image\n","import cv2\n","\n","import matplotlib.pyplot as plt\n","\n","from sklearn.model_selection import train_test_split\n","from sklearn.utils import shuffle\n","\n","import numpy as np\n","import pandas as pd\n","import os\n","import os.path as pth\n","import time\n","from tqdm import tqdm\n","\n","import rdkit\n","from rdkit import Chem\n","from rdkit.Chem import Draw\n","from rdkit import DataStructs\n","from rdkit import RDLogger\n","RDLogger.DisableLog('rdApp.*')  \n","\n","from IPython.display import clear_output\n","\n","from multiprocessing import Process, Queue\n","import datetime\n","import gc\n","\n","import tensorflow.keras as keras\n","from keras.models import Model, Input, load_model\n","from keras.layers import Conv2D, Dense, MaxPooling2D, AveragePooling2D, GlobalAveragePooling2D\n","from keras.layers import Activation, BatchNormalization\n","from keras.layers import Concatenate\n","from keras.utils import to_categorical\n","from keras.callbacks import Callback\n","from keras.optimizers import SGD\n","\n","import numpy as np\n","import keras.backend as K\n","\n","import requests\n","\n","from multiprocessing import Pool\n","from functools import partial\n","\n","import zipfile\n","from google.colab import drive"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"gybLYhALmRNH"},"source":["train_dataset_name = 'all_train_fold_02.tfrecords'\n","val_dataset_name = 'all_val_fold_02.tfrecords'\n","\n","train_another_dataset_name = 'all_train_another_fold_02.tfrecords'\n","val_another_dataset_name = 'all_val_another_fold_02.tfrecords'\n","\n","train_over70_dataset_name = 'all_train_over70_fold_02.tfrecords'\n","val_over70_dataset_name = 'all_val_over70_fold_02.tfrecords'\n","\n","train_under_50per_dataset_name = 'all_train_under_50per_fold_02.tfrecords'\n","val_under_50per_dataset_name = 'all_val_under_50per_fold_02.tfrecords'\n","\n","test_dataset_name = 'test.tfrecords'\n","\n","train_csv_name = 'train.csv'\n","sample_submission_name = 'sample_submission.csv'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TfNgpl27lCn6"},"source":["data_base_path = 'data'\n","train_path = pth.join(data_base_path, 'train')\n","\n","with open(pth.join('data', 'train.csv'), 'r') as csv_file:\n","    data = csv_file.read()\n","    \n","all_captions = []\n","all_img_name_vector = []\n","\n","for line in data.split('\\n')[1:-1]:\n","    image_id, smiles = line.split(',')\n","    caption = '<' + smiles + '>'\n","    full_image_path = pth.join(train_path, image_id)\n","\n","    all_img_name_vector.append(full_image_path)\n","    all_captions.append(caption)\n","\n","train_captions, img_name_vector = shuffle(all_captions, all_img_name_vector, random_state=42)\n","\n","num_examples = 908765 # 학습에 사용할 데이터 수.\n","train_captions = np.array(train_captions[:num_examples])\n","img_name_vector = np.array(img_name_vector[:num_examples])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"6SFdMxb-IsBm"},"source":["옵티마이저를 sgd를 사용할 것이냐를 정하는 변수입니다.  \n","모델이 일정 이상으로 피팅되었다면 sgd를 사용하는 방식으로 사용하였습니다."]},{"cell_type":"code","metadata":{"id":"F3Ko2kCQH6_w"},"source":["is_sgd = False"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"tniODVRNIdRX"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"o_jQGvEFIv49"},"source":["모델의 세부사항을 정하는 설정입니다.\n","이 단계에서 인코더를 어떤 모델을 쓸 지, 디코더의 세부 하이퍼 파라미터는 어떻게 잡을 지를 결정합니다."]},{"cell_type":"code","metadata":{"id":"S-HuX6dRIdOT"},"source":["encoder_base_channel = 8\n","encoder_model_base = 'CustomDenseNet-121'\n","encoder_model_name = 'Autoencoder_{}_trts_basech_{:03d}'.format(encoder_model_base, encoder_base_channel)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Bnq7bSBSI3d_"},"source":["encoder_model_base_path = pth.join(data_base_path, 'checkpoint')\n","encoder_model_path = pth.join(encoder_model_base_path, encoder_model_name)\n","encoder_model_gdrive_path = pth.join(google_drive_base_path, 'model', 'checkpoint', encoder_model_name)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"j2JoqlVs-tfo"},"source":["os.makedirs(encoder_model_path, exist_ok=True)\n","target_checkpoint_filename = sorted(os.listdir(encoder_model_gdrive_path))[-1]\n","encoder_model_filename = pth.join(encoder_model_path, target_checkpoint_filename)\n","encoder_model_gdrive_filename = pth.join(encoder_model_gdrive_path, target_checkpoint_filename)\n","\n","if not pth.exists(encoder_model_filename):\n","    os.system('cp {} {}'.format(encoder_model_gdrive_filename, encoder_model_filename))\n","    while os.path.getsize(encoder_model_gdrive_filename) != os.path.getsize(encoder_model_filename):\n","        os.system('cp {} {}'.format(encoder_model_gdrive_filename, encoder_model_filename))  "],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"B50qrv1RI3bS"},"source":["n_mht = 512\n","n_layer = 4\n","n_dff = 1024\n","n_head = 8\n","# dropout = 0.1\n","dropout = 0\n","decoder_model_name = 'trfrm_mht_{}_layer_{}_dff_{}_head_{}_DO_{}'.format(\n","    n_mht, n_layer, n_dff, n_head, dropout\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"jQA7Y9pUI3ZJ"},"source":["model_name = 'enc-tr_{}_dec-tr_{}_len-100-all-pseudolabel'.format(encoder_model_name, decoder_model_name)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"bvPDl1l_Jdp4"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"IOx2Rt_4JjUJ"},"source":["트랜스포머의 장점은 아무래도 포지셔널 인코딩을 처음에 넉넉한 길이로 잡아놓는다면 이후에 더 긴 분자식도 예측할 수 있다는 점입니다.  \n","기존 베이스라인은 70으로 되어있었지만, 차후 더 복잡한 길이의 예측을 위해서 100으로 잡았습니다."]},{"cell_type":"code","metadata":{"id":"gpcjr4gYJdsK"},"source":["def calc_max_length(tensor):\n","    return max(len(t) for t in tensor)\n","    \n","# max_length = calc_max_length(all_captions)\n","max_length = 100"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"6atIvvMnLREY"},"source":["워드 임베딩 또한 등장할 수 있는 모든 글자를 포함하였습니다."]},{"cell_type":"code","metadata":{"id":"MHcSmaxsJdux"},"source":["tokenizer = tf.keras.preprocessing.text.Tokenizer(lower=False, char_level=True)\n","# temp_captions = all_origin_captions + [\" ^#%()+-.0123456789=@ABCDEFGHIKLMNOPRSTVXYZ[\\\\]abcdefgilmnoprstuy$\"]\n","# tokenizer.fit_on_texts(temp_captions)\n","all_token_list = [\n","    'c', 'C', '(', ')', '1', 'O', '=', '2', 'N', '<', '>', 'n', '[',\n","    ']', '3', '@', 'H', 'l', 'S', '-', 'F', '+', '4', 's', 'o', '#',\n","    'B', 'r', '.', '/', 'P', 'i', 'I', '5', '\\\\', 'e', 'A', 'a', 'g',\n","    '6', 'u', 't', 'T', 'M', 'b', 'K', 'Z', '8', 'd', '9', 'R', 'G',\n","    '7', 'L', 'V', 'h', 'W', 'p', 'm', 'E', 'Y', '0', 'U', 'f', 'D',\n","    'y', 'k', 'X', ' ', '^', '%', '$'\n","]\n","tokenizer.fit_on_texts(all_token_list)\n","top_k = len(tokenizer.word_index)\n","train_seqs = tokenizer.texts_to_sequences(train_captions)\n","cap_vector = tf.keras.preprocessing.sequence.pad_sequences(\n","    train_seqs, maxlen=max_length, padding='post'\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"g1jCS7mpLPsr"},"source":["img_name_train, img_name_val, cap_train, cap_val = train_test_split(img_name_vector, cap_vector, test_size=0.2, random_state=42)\n","len(img_name_train), len(cap_train), len(img_name_val), len(cap_val)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"3sAlD-0DLcSi"},"source":["하이퍼 파라미터 및 학습에 필요한 변수 지정"]},{"cell_type":"code","metadata":{"id":"UDTW-Fy1LPy0"},"source":["BATCH_SIZE = 80\n","BUFFER_SIZE = 100\n","d_model = n_mht\n","num_layers = n_layer\n","dff = n_dff\n","num_heads = n_head\n","vocab_size = top_k # + 1\n","dropout_rate = dropout\n","\n","\n","origin_len = len(img_name_vector)\n","if train_dataset_name.startswith('half_'):\n","    origin_len = origin_len // 2\n","elif train_dataset_name.startswith('quat_'):\n","    origin_len = origin_len // 4\n","\n","another_len = 1000000\n","if train_another_dataset_name.startswith('half_'):\n","    another_len = another_len // 2\n","elif train_another_dataset_name.startswith('quat_'):\n","    another_len = another_len // 4\n","\n","over70_len = 300000\n","if train_over70_dataset_name.startswith('half_'):\n","    over70_len = over70_len // 2\n","elif train_over70_dataset_name.startswith('quat_'):\n","    over70_len = over70_len // 4\n","\n","under50_len = 300000\n","if train_under_50per_dataset_name.startswith('half_'):\n","    under50_len = under50_len // 2\n","elif train_under_50per_dataset_name.startswith('quat_'):\n","    under50_len = under50_len // 4\n","\n","origin_train_len = origin_len//5*4\n","origin_val_len = origin_len//5*1\n","another_train_len = another_len//5*4\n","another_val_len = another_len//5*1\n","over70_train_len = over70_len//5*4\n","over70_val_len = over70_len//5*1\n","under50_train_len = under50_len//5*4\n","under50_val_len = under50_len//5*1\n","\n","train_num_steps = int(np.ceil((origin_train_len+another_train_len+over70_train_len+under50_train_len)/BATCH_SIZE))\n","# train_num_steps = int(np.ceil((origin_train_len+another_train_len)/BATCH_SIZE))\n","val_num_steps = int(np.ceil((origin_val_len+another_val_len+over70_val_len+under50_val_len)/BATCH_SIZE))\n","# val_num_steps = int(np.ceil((origin_val_len+another_val_len)/BATCH_SIZE))\n","\n","# train_num_steps += val_num_steps"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"u4CJe0UeLQAm"},"source":["EPOCHS = 200\n","learning_rate = 1e-4"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"m0KMHLI5Lkm_"},"source":["데이터셋 정의 함수"]},{"cell_type":"code","metadata":{"id":"XgLuw5rqLP9p"},"source":["image_feature_description = {\n","#     'height': tf.io.FixedLenFeature([], tf.int64),\n","#     'width': tf.io.FixedLenFeature([], tf.int64),\n","    'image_raw': tf.io.FixedLenFeature([], tf.string),\n","    # 'label': tf.io.FixedLenFeature([72], tf.float32),\n","    'label_100': tf.io.FixedLenFeature([100], tf.float32),\n","    # 'label_origin': tf.io.FixedLenFeature([], tf.string),\n","    # 'filename': tf.io.FixedLenFeature([], tf.string),\n","}\n","\n","def _parse_image_function(example_proto):\n","    return tf.io.parse_single_example(example_proto, image_feature_description)\n","\n","def map_func(target_record):\n","    img = target_record['image_raw']\n","    cap = target_record['label_100']\n","    # img = tf.io.read_file(image_path)\n","    img = tf.image.decode_jpeg(img, channels=3)\n","    img = tf.dtypes.cast(img, tf.float32)\n","    cap = tf.dtypes.cast(cap, tf.int64)\n","#     img = tf.image.resize(img, (300, 300))\n","    return img, cap\n","\n","def prep_func(image, cap):\n","    result_image = image\n","    return result_image, cap"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"JOYlAre7LPvZ"},"source":["train_dataset_name_list = [\n","                           train_dataset_name, \n","                           train_another_dataset_name, \n","                           train_over70_dataset_name,\n","                           train_under_50per_dataset_name,\n","                           ]\n","train_dataset_name_list = list(map(lambda x: pth.join(data_base_path, x), train_dataset_name_list))\n","\n","val_dataset_name_list = [\n","                         val_dataset_name, \n","                         val_another_dataset_name, \n","                         val_over70_dataset_name,\n","                         val_under_50per_dataset_name,\n","                         ]\n","val_dataset_name_list = list(map(lambda x: pth.join(data_base_path, x), val_dataset_name_list))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"k8KMfEq3Lxnr"},"source":["dataset = tf.data.Dataset.list_files(\n","    train_dataset_name_list\n",")\n","dataset = dataset.interleave(\n","    lambda target_dataset_name: tf.data.TFRecordDataset(target_dataset_name, compression_type='GZIP'),\n","    cycle_length=tf.data.experimental.AUTOTUNE,\n","    num_parallel_calls=tf.data.experimental.AUTOTUNE\n",")\n","dataset = dataset.map(_parse_image_function, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","# dataset = dataset.cache()\n","dataset = dataset.map(map_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","dataset = dataset.shuffle(BUFFER_SIZE)\n","dataset = dataset.batch(BATCH_SIZE)\n","# dataset = dataset.map(prep_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"_Gz-nsvQLxv9"},"source":["dataset_val = tf.data.Dataset.list_files(\n","    val_dataset_name_list\n",")\n","dataset_val = dataset_val.interleave(\n","    lambda target_dataset_name: tf.data.TFRecordDataset(target_dataset_name, compression_type='GZIP'),\n","    cycle_length=tf.data.experimental.AUTOTUNE,\n","    num_parallel_calls=tf.data.experimental.AUTOTUNE\n",")\n","dataset_val = dataset_val.map(_parse_image_function, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","# dataset_val = dataset_val.cache()\n","dataset_val = dataset_val.map(map_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","# dataset_val = dataset_val.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n","dataset_val = dataset_val.batch(BATCH_SIZE)\n","# dataset_val = dataset_val.map(prep_func, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","dataset_val = dataset_val.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0PR7wW6LLyA3"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"noPru-ocMCjT"},"source":["모델 구축"]},{"cell_type":"code","metadata":{"id":"7LyQq65ZLx9B","executionInfo":{"status":"error","timestamp":1602478950032,"user_tz":-540,"elapsed":698,"user":{"displayName":"lww bspl_2","photoUrl":"","userId":"17085094956192153852"}},"outputId":"61b43784-6d0e-45b7-c0e5-a6d588ef1ecb","colab":{"base_uri":"https://localhost:8080/","height":231}},"source":["def get_angles(pos, i, d_model):\n","    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n","    return pos * angle_rates\n","\n","\n","def positional_encoding(position, d_model):\n","    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n","                          np.arange(d_model)[np.newaxis, :],\n","                          d_model)\n","    # apply sin to even indices in the array; 2i\n","    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n","    # apply cos to odd indices in the array; 2i+1\n","    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n","    pos_encoding = angle_rads[np.newaxis, ...]\n","\n","    return tf.cast(pos_encoding, dtype=tf.float32)\n","\n","@tf.function\n","def create_padding_mask(seq):\n","    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n","\n","    # add extra dimensions to add the padding\n","    # to the attention logits.\n","    return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)\n","\n","@tf.function\n","def create_look_ahead_mask(size):\n","    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n","    return mask  # (seq_len, seq_len)\n","\n","\n","def scaled_dot_product_attention(q, k, v, mask):\n","    matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n","\n","    # scale matmul_qk\n","    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n","    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n","\n","    # add the mask to the scaled tensor.\n","    if mask is not None:\n","        scaled_attention_logits += (mask * -1e9)  \n","\n","    # softmax is normalized on the last axis (seq_len_k) so that the scores\n","    # add up to 1.\n","    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n","\n","    output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n","\n","    return output, attention_weights    "],"execution_count":null,"outputs":[{"output_type":"error","ename":"NameError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-1-21f3893541d8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpos_encoding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0;34m@\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcreate_padding_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0mseq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mequal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'tf' is not defined"]}]},{"cell_type":"code","metadata":{"id":"v2c0xQRDLxsl"},"source":["class MultiHeadAttention(tf.keras.layers.Layer):\n","    def __init__(self, d_model, num_heads):\n","        super(MultiHeadAttention, self).__init__()\n","        self.num_heads = num_heads\n","        self.d_model = d_model\n","\n","        assert d_model % self.num_heads == 0\n","\n","        self.depth = d_model // self.num_heads\n","\n","        self.wq = tf.keras.layers.Dense(d_model)\n","        self.wk = tf.keras.layers.Dense(d_model)\n","        self.wv = tf.keras.layers.Dense(d_model)\n","\n","        self.dense = tf.keras.layers.Dense(d_model)\n","\n","    def split_heads(self, x, batch_size):\n","        \"\"\"Split the last dimension into (num_heads, depth).\n","        Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n","        \"\"\"\n","        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n","        return tf.transpose(x, perm=[0, 2, 1, 3])\n","\n","    def call(self, v, k, q, mask):\n","        batch_size = tf.shape(q)[0]\n","\n","        q = self.wq(q)  # (batch_size, seq_len, d_model)\n","        k = self.wk(k)  # (batch_size, seq_len, d_model)\n","        v = self.wv(v)  # (batch_size, seq_len, d_model)\n","\n","        q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n","        k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n","        v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n","\n","        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n","        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n","        scaled_attention, attention_weights = scaled_dot_product_attention(\n","            q, k, v, mask)\n","\n","        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n","\n","        concat_attention = tf.reshape(scaled_attention, \n","                                      (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n","\n","        output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n","\n","        return output, attention_weights\n","\n","\n","def point_wise_feed_forward_network(d_model, dff):\n","    return tf.keras.Sequential([\n","      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n","      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n","    ])\n","\n","\n","class DecoderLayer(tf.keras.layers.Layer):\n","    def __init__(self, d_model, num_heads, dff, rate=0.1):\n","        super(DecoderLayer, self).__init__()\n","\n","        self.mha1 = MultiHeadAttention(d_model, num_heads)\n","        self.mha2 = MultiHeadAttention(d_model, num_heads)\n","\n","        self.ffn = point_wise_feed_forward_network(d_model, dff)\n","\n","        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n","        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n","        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n","\n","        self.dropout1 = tf.keras.layers.Dropout(rate)\n","        self.dropout2 = tf.keras.layers.Dropout(rate)\n","        self.dropout3 = tf.keras.layers.Dropout(rate)\n","\n","\n","    def call(self, x, enc_output, training, \n","           look_ahead_mask, padding_mask):\n","        # enc_output.shape == (batch_size, input_seq_len, d_model)\n","\n","        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n","        attn1 = self.dropout1(attn1, training=training)\n","        out1 = self.layernorm1(attn1 + x)\n","\n","        attn2, attn_weights_block2 = self.mha2(\n","            enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n","        attn2 = self.dropout2(attn2, training=training)\n","        out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n","\n","        ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n","        ffn_output = self.dropout3(ffn_output, training=training)\n","        out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n","\n","        return out3, attn_weights_block1, attn_weights_block2 \n","\n","\n","class Decoder(tf.keras.layers.Layer):\n","    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,\n","               maximum_position_encoding, rate=0.1):\n","        super(Decoder, self).__init__()\n","\n","        self.d_model = d_model\n","        self.num_layers = num_layers\n","\n","        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n","        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n","\n","        self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) \n","                           for _ in range(num_layers)]\n","        self.dropout = tf.keras.layers.Dropout(rate)\n","\n","    def call(self, x, enc_output, training, \n","           look_ahead_mask, padding_mask):\n","        seq_len = tf.shape(x)[1]\n","        attention_weights = {}\n","\n","        x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n","        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n","        x += self.pos_encoding[:, :seq_len, :]\n","\n","        x = self.dropout(x, training=training)\n","\n","        for i in range(self.num_layers):\n","            x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n","                                                 look_ahead_mask, padding_mask)\n","\n","            attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n","            attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n","\n","        # x.shape == (batch_size, target_seq_len, d_model)\n","        return x, attention_weights\n","\n","\n","class CNN_Encoder(tf.keras.Model):\n","    def __init__(self, embedding_dim):\n","        super(CNN_Encoder, self).__init__()\n","        target_checkpoint_filename = sorted(os.listdir(encoder_model_path))[-1]\n","        image_autoencoder = load_model(pth.join(encoder_model_path, target_checkpoint_filename))\n","        image_features_extract_model = image_autoencoder.get_layer(encoder_model_base)\n","#         image_features_extract_model.trainable = False\n","        self.feature_extract_model = image_features_extract_model\n","        self.fc = tf.keras.layers.Dense(embedding_dim, activation='relu')\n","        \n","    def call(self, x):\n","        x = self.feature_extract_model(x)\n","        x = tf.keras.layers.Reshape((-1, x.shape[3]))(x)\n","        x = self.fc(x)\n","        return x\n","\n","\n","class ImageCaptioningTransformer(tf.keras.Model):\n","    def __init__(self, num_layers, d_model, num_heads, dff,\n","               target_vocab_size, pe_target, rate=0.1):\n","        super(ImageCaptioningTransformer, self).__init__()\n","\n","        self.encoder = CNN_Encoder(d_model)\n","\n","        self.decoder = Decoder(num_layers, d_model, num_heads, dff, \n","                               target_vocab_size, pe_target, rate)\n","\n","        self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n","\n","    def call(self, inp, tar, training, look_ahead_mask, dec_padding_mask):\n","        enc_output = self.encoder(inp)\n","\n","        # dec_output.shape == (batch_size, tar_seq_len, d_model)\n","        dec_output, attention_weights = self.decoder(\n","            tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n","\n","        final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n","\n","        return final_output, attention_weights\n","\n","@tf.function\n","def create_masks(tar):\n","    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n","    dec_target_padding_mask = create_padding_mask(tar)\n","    combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n","\n","    return combined_mask        "],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"eaIflosZMMIM"},"source":["모델 생성 및 컴파일"]},{"cell_type":"code","metadata":{"id":"CfP0sZvMMLaX"},"source":["captioning_transformer = ImageCaptioningTransformer(\n","    num_layers, d_model, num_heads, dff,\n","    vocab_size, pe_target=100,\n","    rate=dropout_rate\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"yV-nl-gOMLXU"},"source":["class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n","    def __init__(self, d_model, warmup_steps=5):\n","        super(CustomSchedule, self).__init__()\n","\n","        self.d_model = d_model\n","        self.d_model = tf.cast(self.d_model, tf.float32)\n","\n","        self.warmup_steps = warmup_steps\n","\n","    def __call__(self, step):\n","        arg1 = tf.math.rsqrt(step)\n","        arg2 = step * (self.warmup_steps ** -1.5)\n","\n","        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2HmYwrIRLxq7"},"source":["learning_rate = 1e-4\n","if is_sgd == True:\n","    optimizer = tf.keras.optimizers.SGD(lr=learning_rate)\n","else:\n","    optimizer = tf.keras.optimizers.Adam(lr=learning_rate)\n","\n","# learning_rate = CustomSchedule(d_model)\n","# optimizer = tf.keras.optimizers.Adam(\n","#     learning_rate, beta_1=0.9, beta_2=0.98, \n","#     epsilon=1e-9\n","# )\n","\n","loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n","    from_logits=True, reduction='none'\n",")\n","\n","@tf.function\n","def loss_function(real, pred):\n","    mask = tf.math.logical_not(tf.math.equal(real, 0))\n","    loss_ = loss_object(real, pred)\n","\n","    mask = tf.cast(mask, dtype=loss_.dtype)\n","    loss_ *= mask\n","\n","    return tf.reduce_sum(loss_)/tf.reduce_sum(mask)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ipw3dEriMXM2"},"source":["Checkpoint   \n","이전에 저장된 checkpoint가 있다면 불러옵니다"]},{"cell_type":"code","metadata":{"id":"avvfQgvNMSxV"},"source":["checkpoint_path = pth.join(google_drive_base_path, 'model', 'checkpoint', model_name)\n","os.makedirs(checkpoint_path, exist_ok=True)\n","ckpt = tf.train.Checkpoint(\n","    captioning_transformer=captioning_transformer, \n","    optimizer=optimizer\n",")\n","ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=25)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-KuQA1EhMTCF"},"source":["start_epoch = 0\n","if ckpt_manager.latest_checkpoint:\n","    start_epoch = int(ckpt_manager.latest_checkpoint.split('-')[-1])\n","    ckpt.restore(ckpt_manager.latest_checkpoint)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Lrc-AZDvNOQb"},"source":["학습 정의"]},{"cell_type":"code","metadata":{"id":"s39PLz71MTRr"},"source":["def calculate_similarity(real, pred):\n","#     pred = np.array(list(map(np.array, pred)))\n","#     pred = np.moveaxis(pred, (0,1,2), (1,0,2))\n","    pred = np.argmax(pred, axis=-1)\n","#     print(real[:5], pred[:5])\n","    real = real.numpy()\n","    \n","    score_list = []\n","    for score_i, (each_pred, each_real) in enumerate(zip(pred, real)): \n","        each_pred = ''.join([tokenizer.index_word.get(mol_i, '') for mol_i in each_pred])\n","        each_pred = each_pred.split('>')[0]\n","        m_pred = Chem.MolFromSmiles(each_pred)\n","        if m_pred == None:\n","            score_list.append(0)\n","            continue\n","        each_real = ''.join([tokenizer.index_word.get(mol_i, '') for mol_i in each_real])\n","        each_real = each_real[1:-1]\n","        m_real = Chem.MolFromSmiles(each_real)\n","        \n","        fp_pred = Chem.RDKFingerprint(m_pred)\n","        fp_real = Chem.RDKFingerprint(m_real)\n","        target_similarity = DataStructs.FingerprintSimilarity(fp_real,fp_pred)\n","        score_list.append(target_similarity)\n","        \n","    return score_list"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"b5qsTYHNMTGh"},"source":["# @tf.function(input_signature=train_step_signature)\n","@tf.function\n","def train_step(img_tensor, target, training=True):\n","    target_inp = target[:, :-1]\n","    target_real = target[:, 1:]\n","    \n","    combined_mask = create_masks(target_inp)\n","    \n","    with tf.GradientTape() as tape:\n","        predictions, _ = captioning_transformer(\n","            inp=img_tensor, tar=target_inp, training=training, \n","            look_ahead_mask=combined_mask, dec_padding_mask=None\n","#             look_ahead_mask=None, dec_padding_mask=None\n","        )\n","        loss = loss_function(target_real, predictions)\n","#         total_loss = (loss / int(target_inp.shape[1]))\n","        if training == True:\n","            gradients = tape.gradient(loss, captioning_transformer.trainable_variables)    \n","            optimizer.apply_gradients(zip(gradients, captioning_transformer.trainable_variables))\n","\n","    train_accuracy(target_real, predictions)\n","    return loss, predictions"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8UqDgMDpMS2z"},"source":["loss_plot, val_loss_plot = [], []\n","sim_plot, val_sim_plot = [], []\n","lowest_val_loss = 1e12\n","train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n","    name='train_accuracy')\n","gc.collect()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"AEJbjfVfMS1A"},"source":["for epoch in range(start_epoch, EPOCHS):\n","    total_loss, total_val_loss, total_test_pred_26_loss = 0, 0, 0\n","    train_accuracy.reset_states()\n","\n","    tqdm_dataset = tqdm(enumerate(dataset), total=train_num_steps, position=0, leave=True)\n","    total_sim = 0\n","    for (batch, (img_tensor, target)) in tqdm_dataset:\n","        valid_cap_mask = (target[:,0] == 10)\n","        img_tensor = img_tensor[valid_cap_mask]\n","        target = target[valid_cap_mask]\n","\n","        batch_loss, pred_list = train_step(img_tensor, target, training=True)\n","        smilarlity_list = calculate_similarity(target, pred_list)\n","        smilarlity = np.mean(smilarlity_list)\n","        total_sim += smilarlity\n","        total_loss += batch_loss\n","        if batch % 50 == 0:\n","            tqdm_dataset.set_postfix({\n","                'Epoch': epoch + 1,\n","                'Batch': batch,\n","                'Loss': '{:06f}'.format(batch_loss.numpy() / int(target.shape[1])),\n","                'Similarlity': smilarlity,\n","                'Accuracy':train_accuracy.result().numpy(), \n","            })\n","        if batch % 30 == 0:\n","            gc.collect()\n","    loss_plot.append(total_loss / (batch+1))\n","    sim_plot.append(total_sim / (batch+1))\n","\n","    tqdm_dataset_val = tqdm(enumerate(dataset_val), total=val_num_steps, position=0, leave=True)\n","    total_val_sim = 0\n","    for (batch, (img_tensor, target)) in tqdm_dataset_val:\n","        valid_cap_mask = (target[:,0] == 10)\n","        img_tensor = img_tensor[valid_cap_mask]\n","        target = target[valid_cap_mask]\n","\n","        batch_val_loss, pred_list = train_step(img_tensor, target, training=False)\n","        smilarlity_list = calculate_similarity(target, pred_list)\n","        smilarlity = np.mean(smilarlity_list)\n","        total_val_sim += smilarlity\n","        total_val_loss += batch_val_loss\n","        if batch % 50 == 0:\n","            tqdm_dataset_val.set_postfix({\n","                'Epoch': epoch + 1,\n","                'Batch': batch,\n","                'Val Loss': '{:06f}'.format(batch_val_loss.numpy() / int(target.shape[1])),\n","                'Var Similarlity': smilarlity,\n","                'Var Accuracy':train_accuracy.result().numpy(), \n","            })\n","        if batch % 30 == 0:\n","            gc.collect()\n","    val_loss_plot.append(total_val_loss / (batch+1))\n","    val_sim_plot.append(total_val_sim / (batch+1))\n","\n","    ckpt_manager.save()\n","\n","    output.clear()\n","\n","    plt.figure()\n","    plt.plot(loss_plot, label='loss')\n","    plt.plot(val_loss_plot, label='val_loss')\n","    plt.xlabel('Epochs')\n","    plt.ylabel('Loss')\n","    plt.title('Loss Plot')\n","    plt.legend()\n","    plt.show()\n","\n","    plt.figure()\n","    plt.plot(sim_plot, label='similarity')\n","    plt.plot(val_sim_plot, label='val_Similarity')\n","    plt.ylim(-0.1,1.1)\n","    plt.xlabel('Epochs')\n","    plt.ylabel('Similarity')\n","    plt.title('Similarity Plot')\n","    plt.legend()\n","    plt.show()\n","\n","    print()\n","    # print ('Epoch {}, Loss {:.6f}, Similiarity {:.6f}'.format(\n","    #     epoch + 1, loss_plot[-1], sim_plot[-1]))    \n","    print ('Epoch {}, Loss {:.6f}, Val loss: {:.6f}, Similiarity {:.6f}, Val similiarity: {:.6f}'.format(\n","        epoch + 1, loss_plot[-1], val_loss_plot[-1], sim_plot[-1], val_sim_plot[-1]))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"DsMrPQIGMSuE"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"DBBYzvf2Lxkh"},"source":[""],"execution_count":null,"outputs":[]}]}