{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-04T11:44:31.108306Z","iopub.execute_input":"2022-11-04T11:44:31.108764Z","iopub.status.idle":"2022-11-04T11:44:31.141427Z","shell.execute_reply.started":"2022-11-04T11:44:31.108674Z","shell.execute_reply":"2022-11-04T11:44:31.140030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport pickle\nfrom PIL import Image, ImageDraw\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.applications import InceptionV3\n\nfrom tensorflow.keras.layers import Dropout, Flatten, Dense, Input\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n!pip install face_recognition\nimport face_recognition","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:49:44.770151Z","iopub.execute_input":"2022-11-04T11:49:44.770531Z","iopub.status.idle":"2022-11-04T11:49:54.031027Z","shell.execute_reply.started":"2022-11-04T11:49:44.770500Z","shell.execute_reply":"2022-11-04T11:49:54.029858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/challenges-in-representation-learning-facial-expression-recognition-challenge/train.csv')\nprint(train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:45:12.738715Z","iopub.execute_input":"2022-11-04T11:45:12.739903Z","iopub.status.idle":"2022-11-04T11:45:17.608749Z","shell.execute_reply.started":"2022-11-04T11:45:12.739860Z","shell.execute_reply":"2022-11-04T11:45:17.607690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View train_df\n\ntrain_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:45:17.610302Z","iopub.execute_input":"2022-11-04T11:45:17.610712Z","iopub.status.idle":"2022-11-04T11:45:17.631848Z","shell.execute_reply.started":"2022-11-04T11:45:17.610675Z","shell.execute_reply":"2022-11-04T11:45:17.631000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a 4D array of pixel values\n\ntrain_df['pixels'] = [np.fromstring(x, dtype=int, sep=' ').reshape(-1,48,48,1) for x in train_df['pixels']]","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:45:17.634235Z","iopub.execute_input":"2022-11-04T11:45:17.634600Z","iopub.status.idle":"2022-11-04T11:45:19.337830Z","shell.execute_reply.started":"2022-11-04T11:45:17.634566Z","shell.execute_reply":"2022-11-04T11:45:19.336802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pixels = np.concatenate(train_df['pixels'])\nlabels = train_df.emotion.values\n\nprint(pixels.shape)\nprint(labels.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:45:19.339414Z","iopub.execute_input":"2022-11-04T11:45:19.339812Z","iopub.status.idle":"2022-11-04T11:45:19.605090Z","shell.execute_reply.started":"2022-11-04T11:45:19.339776Z","shell.execute_reply":"2022-11-04T11:45:19.604067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dist = (train_df.emotion.value_counts() / len(train_df)).to_frame().sort_index(ascending=True).T\n\ntrain_dist","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:45:19.606749Z","iopub.execute_input":"2022-11-04T11:45:19.607438Z","iopub.status.idle":"2022-11-04T11:45:19.628833Z","shell.execute_reply.started":"2022-11-04T11:45:19.607400Z","shell.execute_reply":"2022-11-04T11:45:19.627747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\n\nsns.displot(train_dist, x=train_df.emotion)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:45:19.630382Z","iopub.execute_input":"2022-11-04T11:45:19.630731Z","iopub.status.idle":"2022-11-04T11:45:20.000683Z","shell.execute_reply.started":"2022-11-04T11:45:19.630698Z","shell.execute_reply":"2022-11-04T11:45:19.999736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **View samples of images**","metadata":{}},{"cell_type":"code","source":"emotions = ['Angry','Disgust','Fear','Happy','Sad','Surprise','Neutral']","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:45:20.002165Z","iopub.execute_input":"2022-11-04T11:45:20.002520Z","iopub.status.idle":"2022-11-04T11:45:20.007658Z","shell.execute_reply.started":"2022-11-04T11:45:20.002485Z","shell.execute_reply":"2022-11-04T11:45:20.006421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.close()\nplt.rcParams[\"figure.figsize\"] = [16,16]\n\nrow = 0\nfor emotion in np.unique(labels):\n\n    all_emotion_images = train_df[train_df['emotion'] == emotion]\n    for i in range(5):\n        \n        img = all_emotion_images.iloc[i,].pixels.reshape(48,48)\n        lab = emotions[emotion]\n\n        plt.subplot(7,5,row+i+1)\n        plt.imshow(img, cmap='binary_r')\n        plt.text(-30, 5, s = str(lab), fontsize=10, color='b')\n        plt.axis('off')\n    row += 5\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:45:20.009306Z","iopub.execute_input":"2022-11-04T11:45:20.009669Z","iopub.status.idle":"2022-11-04T11:45:21.713001Z","shell.execute_reply.started":"2022-11-04T11:45:20.009635Z","shell.execute_reply":"2022-11-04T11:45:21.712110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 截取嘴巴\n\n# img = all_emotion_images.iloc[4,].pixels.reshape(48,48)\n# img = Image.fromarray(img.astype(np.uint8))\n# img = np.array(img)\n# face_locations = face_recognition.face_locations(img)\n# face_landmarks = face_recognition.face_landmarks(img)\n# # print(face_landmarks)\n# # print(face_locations)\n# face_landmark = face_landmarks[0]\n# tmp = face_landmark['top_lip']\n# tmp.extend(face_landmark['bottom_lip'])\n\n# def test_show(image):\n#     image = Image.fromarray(image.astype(np.uint8))\n#     return image\n# x = []\n# y = []\n# for tup_x,tup_y in tmp:\n#     x.append(tup_x)\n#     y.append(tup_y)\n# lip_top,lip_right,lip_bottom,lip_left = min(y),max(x),max(y),min(x)\n# print(lip_top,lip_right,lip_bottom,lip_left)\n# face_image = img[lip_top:lip_bottom, lip_left:lip_right]\n# face_image = test_show(face_image)\n# face_image","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:50:14.922579Z","iopub.execute_input":"2022-11-04T11:50:14.922935Z","iopub.status.idle":"2022-11-04T11:50:14.928467Z","shell.execute_reply.started":"2022-11-04T11:50:14.922905Z","shell.execute_reply":"2022-11-04T11:50:14.927124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lip_cut(pixel):\n    img = pixel.reshape(48,48)\n    img = Image.fromarray(img.astype(np.uint8))\n    img = np.array(img)\n    face_landmarks = face_recognition.face_landmarks(img)\n    if not face_landmarks:\n        return None,False\n    face_landmark = face_landmarks[0]\n    tmp = face_landmark['top_lip']\n    tmp.extend(face_landmark['bottom_lip'])\n    x = []\n    y = []\n    for tup_x,tup_y in tmp:\n        x.append(tup_x)\n        y.append(tup_y)\n    lip_top,lip_right,lip_bottom,lip_left = min(y),max(x),max(y),min(x)\n    face_image = img[lip_top:lip_bottom, lip_left:lip_right]\n    face_image = Image.fromarray(face_image.astype(np.uint8))\n    face_image = face_image.resize((48,48))\n    return np.array(face_image),True","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:50:11.859919Z","iopub.execute_input":"2022-11-04T11:50:11.860295Z","iopub.status.idle":"2022-11-04T11:50:11.868746Z","shell.execute_reply.started":"2022-11-04T11:50:11.860265Z","shell.execute_reply":"2022-11-04T11:50:11.867403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 对每个像素进行嘴唇裁剪\npixels_lip,labels_lip = [],[]\nfor p,l in zip(pixels,labels):\n    p,flag = lip_cut(p)\n#     成功检测到人脸\n    if flag:\n        p = p.reshape(48,48,1)\n        pixels_lip.append(p)\n        labels_lip.append(l)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:50:18.458856Z","iopub.execute_input":"2022-11-04T11:50:18.459234Z","iopub.status.idle":"2022-11-04T11:51:28.154952Z","shell.execute_reply.started":"2022-11-04T11:50:18.459203Z","shell.execute_reply":"2022-11-04T11:51:28.153914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(\n    np.array(pixels_lip), np.array(labels_lip), test_size=0.2, stratify=labels_lip, random_state=1\n)\n\n\nprint('X_train Shape:', X_train.shape)\nprint('y_train Shape:', y_train.shape)\nprint()\nprint('X_valid Shape:', X_valid.shape)\nprint('y_valid Shape:', y_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:51:28.156771Z","iopub.execute_input":"2022-11-04T11:51:28.157146Z","iopub.status.idle":"2022-11-04T11:51:28.234681Z","shell.execute_reply.started":"2022-11-04T11:51:28.157111Z","shell.execute_reply":"2022-11-04T11:51:28.233649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs_train = X_train / 255\nXs_valid = X_valid / 255","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:51:28.235987Z","iopub.execute_input":"2022-11-04T11:51:28.237632Z","iopub.status.idle":"2022-11-04T11:51:28.354707Z","shell.execute_reply.started":"2022-11-04T11:51:28.237595Z","shell.execute_reply":"2022-11-04T11:51:28.353653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load base model","metadata":{}},{"cell_type":"code","source":"np.random.seed(1)\ntf.random.set_seed(1)\n\nbase_model = tf.keras.applications.VGG16(\n    input_shape = (48, 48, 1), \n    include_top = False, \n    weights = None #'imagenet'\n)\n\nbase_model.trainable = False\n\ntf.keras.utils.plot_model(base_model, show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:51:28.357297Z","iopub.execute_input":"2022-11-04T11:51:28.358643Z","iopub.status.idle":"2022-11-04T11:51:32.184099Z","shell.execute_reply.started":"2022-11-04T11:51:28.358603Z","shell.execute_reply":"2022-11-04T11:51:32.182867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:51:32.185848Z","iopub.execute_input":"2022-11-04T11:51:32.186662Z","iopub.status.idle":"2022-11-04T11:51:32.198788Z","shell.execute_reply.started":"2022-11-04T11:51:32.186624Z","shell.execute_reply":"2022-11-04T11:51:32.196862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build a new model","metadata":{}},{"cell_type":"code","source":"np.random.seed(1)\ntf.random.set_seed(1)\n\ncnn = Sequential([\n    Conv2D(32, (3,3), activation = 'relu', padding = 'same', input_shape=(48,48,1)),\n    Conv2D(32, (3,3), activation = 'relu', padding = 'same'),\n    MaxPooling2D(2,2),\n    Dropout(0.25),\n    BatchNormalization(),\n    \n    Conv2D(64, (3,3), activation = 'relu', padding = 'same'),\n    Conv2D(64, (3,3), activation = 'relu', padding = 'same'),\n    MaxPooling2D(2,2),\n    Dropout(0.25),\n    BatchNormalization(),\n\n\n    Conv2D(128, (3,3), activation = 'relu', padding = 'same'),\n    Conv2D(128, (3,3), activation = 'relu', padding = 'same'),\n    MaxPooling2D(2,2),\n    Dropout(0.65),\n    BatchNormalization(),\n    \n    Conv2D(64, (3,3), activation = 'relu', padding = 'same'),\n    Conv2D(64, (3,3), activation = 'relu', padding = 'same'),\n    MaxPooling2D(2,2),\n    Dropout(0.25),\n    BatchNormalization(),\n\n    Flatten(),\n    \n    Dense(64, activation='relu'),\n    Dropout(0.25),\n    Dense(128, activation='relu'),\n    Dropout(0.65),\n    Dense(16, activation='relu'),\n    Dropout(0.25),\n    BatchNormalization(),\n    Dense(7, activation='softmax')\n])\n\ncnn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:51:32.201608Z","iopub.execute_input":"2022-11-04T11:51:32.202200Z","iopub.status.idle":"2022-11-04T11:51:32.395057Z","shell.execute_reply.started":"2022-11-04T11:51:32.202165Z","shell.execute_reply":"2022-11-04T11:51:32.394040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Network","metadata":{}},{"cell_type":"code","source":"opt = tf.keras.optimizers.Adam(0.001)\ncnn.compile(loss='sparse_categorical_crossentropy', optimizer=opt, metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:51:32.396636Z","iopub.execute_input":"2022-11-04T11:51:32.396980Z","iopub.status.idle":"2022-11-04T11:51:32.410804Z","shell.execute_reply.started":"2022-11-04T11:51:32.396945Z","shell.execute_reply":"2022-11-04T11:51:32.409811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train run 1\n# %%time \n\nh1 = cnn.fit(\n    Xs_train, y_train, \n    batch_size = 32,\n    epochs = 80,\n    verbose = 1,\n    validation_data = (Xs_valid, y_valid)\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:51:32.412612Z","iopub.execute_input":"2022-11-04T11:51:32.413049Z","iopub.status.idle":"2022-11-04T11:56:45.325379Z","shell.execute_reply.started":"2022-11-04T11:51:32.413014Z","shell.execute_reply":"2022-11-04T11:56:45.324405Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = h1.history\nprint(history.keys())","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:58:16.672902Z","iopub.execute_input":"2022-11-04T11:58:16.673200Z","iopub.status.idle":"2022-11-04T11:58:16.679210Z","shell.execute_reply.started":"2022-11-04T11:58:16.673173Z","shell.execute_reply":"2022-11-04T11:58:16.678329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,2,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,2,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:58:22.491588Z","iopub.execute_input":"2022-11-04T11:58:22.491965Z","iopub.status.idle":"2022-11-04T11:58:22.884195Z","shell.execute_reply.started":"2022-11-04T11:58:22.491933Z","shell.execute_reply":"2022-11-04T11:58:22.883290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train run 2\ntf.keras.backend.set_value(cnn.optimizer.learning_rate, 0.0001)\n\nh2 = cnn.fit(\n    Xs_train, y_train, \n    batch_size=32,\n    epochs = 20,\n    verbose = 1,\n    validation_data = (Xs_valid, y_valid)\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T11:59:13.319922Z","iopub.execute_input":"2022-11-04T11:59:13.320306Z","iopub.status.idle":"2022-11-04T12:00:35.612115Z","shell.execute_reply.started":"2022-11-04T11:59:13.320272Z","shell.execute_reply":"2022-11-04T12:00:35.611071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in history.keys():\n    history[k] += h2.history[k]\n\nepoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,2,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,2,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T12:01:40.158082Z","iopub.execute_input":"2022-11-04T12:01:40.158519Z","iopub.status.idle":"2022-11-04T12:01:40.552297Z","shell.execute_reply.started":"2022-11-04T12:01:40.158483Z","shell.execute_reply":"2022-11-04T12:01:40.551330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train run 3\ntf.keras.backend.set_value(cnn.optimizer.learning_rate, 0.00001)\n\nh3 = cnn.fit(\n    Xs_train, y_train, \n    batch_size=32,\n    epochs = 20,\n    verbose = 1,\n    validation_data = (Xs_valid, y_valid)\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T12:02:05.638503Z","iopub.execute_input":"2022-11-04T12:02:05.638865Z","iopub.status.idle":"2022-11-04T12:03:27.951610Z","shell.execute_reply.started":"2022-11-04T12:02:05.638834Z","shell.execute_reply":"2022-11-04T12:03:27.950605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in history.keys():\n    history[k] += h3.history[k]\n\nepoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,2,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,2,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T12:03:35.220656Z","iopub.execute_input":"2022-11-04T12:03:35.221028Z","iopub.status.idle":"2022-11-04T12:03:35.621398Z","shell.execute_reply.started":"2022-11-04T12:03:35.220996Z","shell.execute_reply":"2022-11-04T12:03:35.620421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# save model","metadata":{}},{"cell_type":"code","source":"cnn.save('fer_model_lip_v01.h5')\npickle.dump(history, open(f'fer_v01.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-11-04T12:05:00.945768Z","iopub.execute_input":"2022-11-04T12:05:00.946135Z","iopub.status.idle":"2022-11-04T12:05:01.042824Z","shell.execute_reply.started":"2022-11-04T12:05:00.946104Z","shell.execute_reply":"2022-11-04T12:05:01.041847Z"},"trusted":true},"execution_count":null,"outputs":[]}]}