{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":77476,"databundleVersionId":8462666,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:01:58.386296Z","iopub.execute_input":"2024-05-14T18:01:58.387378Z","iopub.status.idle":"2024-05-14T18:01:58.394885Z","shell.execute_reply.started":"2024-05-14T18:01:58.387333Z","shell.execute_reply":"2024-05-14T18:01:58.393718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_file_path = '/kaggle/input/2024-smarcle-ks-3-fashion-mnist2/train.csv'\ntest_file_path = '/kaggle/input/2024-smarcle-ks-3-fashion-mnist2/test.csv'\ntrain_data = pd.read_csv(train_file_path)\ntest_data = pd.read_csv(test_file_path)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:01:58.396775Z","iopub.execute_input":"2024-05-14T18:01:58.397122Z","iopub.status.idle":"2024-05-14T18:02:03.721496Z","shell.execute_reply.started":"2024-05-14T18:01:58.397090Z","shell.execute_reply":"2024-05-14T18:02:03.720649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 0 Let's go~","metadata":{}},{"cell_type":"markdown","source":"오랜만에 하는 캐글스터디... 화이팅 해봅시다.","metadata":{}},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:03.722781Z","iopub.execute_input":"2024-05-14T18:02:03.723141Z","iopub.status.idle":"2024-05-14T18:02:03.743112Z","shell.execute_reply.started":"2024-05-14T18:02:03.723096Z","shell.execute_reply":"2024-05-14T18:02:03.741974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:03.744453Z","iopub.execute_input":"2024-05-14T18:02:03.744840Z","iopub.status.idle":"2024-05-14T18:02:03.773569Z","shell.execute_reply.started":"2024-05-14T18:02:03.744810Z","shell.execute_reply":"2024-05-14T18:02:03.772436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.shape,test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:03.777205Z","iopub.execute_input":"2024-05-14T18:02:03.777541Z","iopub.status.idle":"2024-05-14T18:02:03.784695Z","shell.execute_reply.started":"2024-05-14T18:02:03.777512Z","shell.execute_reply":"2024-05-14T18:02:03.783565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.describe()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:03.785744Z","iopub.execute_input":"2024-05-14T18:02:03.786076Z","iopub.status.idle":"2024-05-14T18:02:06.427394Z","shell.execute_reply.started":"2024-05-14T18:02:03.786013Z","shell.execute_reply":"2024-05-14T18:02:06.426380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Each pixel is composed of 28 * 28. \n### But why 28 * 28 = 785? \n### Becasue there is a label at first! \n### Let's seperate them","metadata":{}},{"cell_type":"code","source":"x_train = train_data.drop(columns=[\"label\"])\ny_train = train_data[\"label\"]\nx_test = test_data\nprint(x_train.shape)\nprint(y_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:06.428888Z","iopub.execute_input":"2024-05-14T18:02:06.429557Z","iopub.status.idle":"2024-05-14T18:02:06.547694Z","shell.execute_reply.started":"2024-05-14T18:02:06.429516Z","shell.execute_reply":"2024-05-14T18:02:06.546725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:06.549041Z","iopub.execute_input":"2024-05-14T18:02:06.549425Z","iopub.status.idle":"2024-05-14T18:02:06.554495Z","shell.execute_reply.started":"2024-05-14T18:02:06.549398Z","shell.execute_reply":"2024-05-14T18:02:06.553550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Count how many of each label","metadata":{}},{"cell_type":"code","source":"sns.countplot(x=y_train)\nplt.show()\n\n# 레이블 발생 횟수 출력\nprint(list(y_train.value_counts().sort_index()))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:06.555801Z","iopub.execute_input":"2024-05-14T18:02:06.556209Z","iopub.status.idle":"2024-05-14T18:02:06.784153Z","shell.execute_reply.started":"2024-05-14T18:02:06.556182Z","shell.execute_reply":"2024-05-14T18:02:06.783109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### We can know that each label has 6000 instances","metadata":{}},{"cell_type":"markdown","source":"# 1. Preprocessing","metadata":{}},{"cell_type":"markdown","source":"### 1-1 Check missing values","metadata":{}},{"cell_type":"code","source":"missing_values = x_train.isnull().sum()\n\n# 결측값이 있는 열을 출력합니다.\nprint(\"Coulmns with missing values ::\")\nprint(missing_values[missing_values > 0])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:06.785472Z","iopub.execute_input":"2024-05-14T18:02:06.785756Z","iopub.status.idle":"2024-05-14T18:02:06.827212Z","shell.execute_reply.started":"2024-05-14T18:02:06.785730Z","shell.execute_reply":"2024-05-14T18:02:06.826157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"no missing values!!!","metadata":{}},{"cell_type":"markdown","source":"### 1-2 Convert form","metadata":{}},{"cell_type":"markdown","source":"##### Let's change pandas DataFrame to numpy because numpy is more compatible than pandas in image processing","metadata":{}},{"cell_type":"code","source":"x_train = x_train.to_numpy()\ny_train = y_train.to_numpy()\nx_test = x_test.to_numpy()\nprint(x_train[0])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:06.828555Z","iopub.execute_input":"2024-05-14T18:02:06.828902Z","iopub.status.idle":"2024-05-14T18:02:06.837212Z","shell.execute_reply.started":"2024-05-14T18:02:06.828874Z","shell.execute_reply":"2024-05-14T18:02:06.836254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train.reshape(x_train.shape[0], 28, 28) # x_train.shape[0] same -1","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:06.838479Z","iopub.execute_input":"2024-05-14T18:02:06.838758Z","iopub.status.idle":"2024-05-14T18:02:06.846976Z","shell.execute_reply.started":"2024-05-14T18:02:06.838734Z","shell.execute_reply":"2024-05-14T18:02:06.846095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(6, 6))\nfor i in range(20):\n    ax = fig.add_subplot(5, 5, i + 1, xticks=[], yticks=[])\n    ax.imshow(x_train[i], cmap=plt.cm.binary, interpolation='nearest')\n    # label the image with the target value\n    ax.text(0, 7, str(y_train[i]))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:06.848251Z","iopub.execute_input":"2024-05-14T18:02:06.849512Z","iopub.status.idle":"2024-05-14T18:02:07.894458Z","shell.execute_reply.started":"2024-05-14T18:02:06.849476Z","shell.execute_reply":"2024-05-14T18:02:07.893386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### 0 T-shirt/top  1 Trouser    2 Pullover     3 Dress     4 Coat     5 Sandal     6 Shirt      7 Sneaker     8 Bag      9 Ankle boot","metadata":{}},{"cell_type":"markdown","source":"#### In Keras, to use datas effectively, change the numbers in the range of 0 to 255 to the range of 0 to 1. (Normalization)","metadata":{}},{"cell_type":"code","source":"x_train = x_train.reshape(x_train.shape[0], -1)\nx_test = x_test.reshape(x_test.shape[0],-1)\n\nx_train = x_train.astype('float64') # To divide the number, change the number from integer to float. \nx_train = x_train / 255\n\nx_test = x_test.astype('float64') \nx_test = x_test / 255\n\nprint(x_train[0])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:07.898846Z","iopub.execute_input":"2024-05-14T18:02:07.899225Z","iopub.status.idle":"2024-05-14T18:02:08.192444Z","shell.execute_reply.started":"2024-05-14T18:02:07.899197Z","shell.execute_reply":"2024-05-14T18:02:08.191357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1-3 PCA Mnist datas","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:08.193941Z","iopub.execute_input":"2024-05-14T18:02:08.194357Z","iopub.status.idle":"2024-05-14T18:02:08.199097Z","shell.execute_reply.started":"2024-05-14T18:02:08.194320Z","shell.execute_reply":"2024-05-14T18:02:08.198166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_train.shape)\nprint(x_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:08.200544Z","iopub.execute_input":"2024-05-14T18:02:08.201376Z","iopub.status.idle":"2024-05-14T18:02:08.210658Z","shell.execute_reply.started":"2024-05-14T18:02:08.201337Z","shell.execute_reply":"2024-05-14T18:02:08.209477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=.95)\nx_train_pca = pca.fit_transform(x_train)\nx_test_pca = pca.fit_transform(x_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:08.211882Z","iopub.execute_input":"2024-05-14T18:02:08.212174Z","iopub.status.idle":"2024-05-14T18:02:15.698107Z","shell.execute_reply.started":"2024-05-14T18:02:08.212142Z","shell.execute_reply":"2024-05-14T18:02:15.696569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Total number of components used after PCA : {pca.n_components_}')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:15.700114Z","iopub.execute_input":"2024-05-14T18:02:15.700840Z","iopub.status.idle":"2024-05-14T18:02:15.706178Z","shell.execute_reply.started":"2024-05-14T18:02:15.700798Z","shell.execute_reply":"2024-05-14T18:02:15.705235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_train_pca.shape)\nprint(x_test_pca.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:15.707914Z","iopub.execute_input":"2024-05-14T18:02:15.708539Z","iopub.status.idle":"2024-05-14T18:02:15.718843Z","shell.execute_reply.started":"2024-05-14T18:02:15.708503Z","shell.execute_reply":"2024-05-14T18:02:15.717564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see 784 -> 187","metadata":{}},{"cell_type":"code","source":"# PCA 모델 훈련\n# 변환된 데이터 시각화\nplt.figure(figsize=(8, 6))\nplt.scatter(x_train_pca[:, 0], x_train_pca[:, 1], c=y_train, cmap='viridis', s=10)\nplt.colorbar(label='digit label', ticks=range(10))\nplt.xlabel('Principal Component 1')\nplt.ylabel('Principal Component 2')\nplt.title('PCA Visualization of MNIST Dataset')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:15.720732Z","iopub.execute_input":"2024-05-14T18:02:15.721159Z","iopub.status.idle":"2024-05-14T18:02:17.072089Z","shell.execute_reply.started":"2024-05-14T18:02:15.721121Z","shell.execute_reply":"2024-05-14T18:02:17.071094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1-4 One-hot-encoding","metadata":{}},{"cell_type":"code","source":"from keras.utils import to_categorical\nprint(\"before one-hot-encoding \",y_train[0])\ny_train = to_categorical(y_train)\nprint(\"after one-hot-encoding \",y_train[0])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:17.073423Z","iopub.execute_input":"2024-05-14T18:02:17.073783Z","iopub.status.idle":"2024-05-14T18:02:17.082135Z","shell.execute_reply.started":"2024-05-14T18:02:17.073745Z","shell.execute_reply":"2024-05-14T18:02:17.081083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"할게 이거말고 더 있을 것 같은데..","metadata":{}},{"cell_type":"markdown","source":"# 2 Modeling","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:17.083836Z","iopub.execute_input":"2024-05-14T18:02:17.084195Z","iopub.status.idle":"2024-05-14T18:02:17.092285Z","shell.execute_reply.started":"2024-05-14T18:02:17.084167Z","shell.execute_reply":"2024-05-14T18:02:17.091207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train.reshape((x_train.shape[0], 28 ,28, 1))\nx_test = x_test.reshape((x_test.shape[0], 28 ,28, 1))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:17.093821Z","iopub.execute_input":"2024-05-14T18:02:17.094243Z","iopub.status.idle":"2024-05-14T18:02:17.104144Z","shell.execute_reply.started":"2024-05-14T18:02:17.094205Z","shell.execute_reply":"2024-05-14T18:02:17.103209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2-1 Split train in train and validation set","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, Y_train, Y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=28)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:17.105573Z","iopub.execute_input":"2024-05-14T18:02:17.105895Z","iopub.status.idle":"2024-05-14T18:02:17.690895Z","shell.execute_reply.started":"2024-05-14T18:02:17.105870Z","shell.execute_reply":"2024-05-14T18:02:17.689732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Fashion MNIST train -  rows:\",X_train.shape[0],\" columns:\", X_train.shape[1:4])\nprint(\"Fashion MNIST valid -  rows:\",X_val.shape[0],\" columns:\", X_val.shape[1:4])\nprint(\"Fashion MNIST test -  rows:\",x_test.shape[0],\" columns:\", x_test.shape[1:4])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:17.692375Z","iopub.execute_input":"2024-05-14T18:02:17.692712Z","iopub.status.idle":"2024-05-14T18:02:17.699447Z","shell.execute_reply.started":"2024-05-14T18:02:17.692664Z","shell.execute_reply":"2024-05-14T18:02:17.698313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2-2 Creating and compiling a model","metadata":{}},{"cell_type":"code","source":"import os\n\nmodel = tf.keras.Sequential([\n        tf.keras.layers.Conv2D(32, kernel_size=(3, 3),\n                               activation='relu',\n                               kernel_initializer='he_normal',\n                               input_shape=(28, 28, 1)),\n        tf.keras.layers.MaxPooling2D((2, 2)),\n        tf.keras.layers.Conv2D(64, \n                               kernel_size=(3, 3), \n                               activation='relu'),\n        tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n        tf.keras.layers.Conv2D(128, (3, 3), activation='relu'),\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(128, activation='relu'),\n        tf.keras.layers.Dense(10, activation='softmax')\n    ])\n# 모델 컴파일\nmodel.compile(loss=tf.keras.losses.categorical_crossentropy,\n                optimizer='adam',\n                metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:17.700758Z","iopub.execute_input":"2024-05-14T18:02:17.701176Z","iopub.status.idle":"2024-05-14T18:02:17.767408Z","shell.execute_reply.started":"2024-05-14T18:02:17.701140Z","shell.execute_reply":"2024-05-14T18:02:17.766239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2-3 Model summary","metadata":{}},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:17.768652Z","iopub.execute_input":"2024-05-14T18:02:17.768943Z","iopub.status.idle":"2024-05-14T18:02:17.794552Z","shell.execute_reply.started":"2024-05-14T18:02:17.768916Z","shell.execute_reply":"2024-05-14T18:02:17.793618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.utils import plot_model\n\n# 모델 시각화\nplot_model(model, to_file='model.png', show_shapes=True, show_layer_names=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:17.795704Z","iopub.execute_input":"2024-05-14T18:02:17.796001Z","iopub.status.idle":"2024-05-14T18:02:18.032188Z","shell.execute_reply.started":"2024-05-14T18:02:17.795975Z","shell.execute_reply":"2024-05-14T18:02:18.031107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2-4 Model fit","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\n# EarlyStopping 콜백 설정\nearly_stopping_callback = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)\n\n# 모델 훈련\ntrain_model = model.fit(X_train, Y_train,\n                        batch_size=200,\n                        epochs=40,\n                        verbose=1,\n                        validation_data=(X_val, Y_val),\n                        callbacks=[early_stopping_callback])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:18.033486Z","iopub.execute_input":"2024-05-14T18:02:18.033781Z","iopub.status.idle":"2024-05-14T18:02:38.199141Z","shell.execute_reply.started":"2024-05-14T18:02:18.033754Z","shell.execute_reply":"2024-05-14T18:02:38.198229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(x_test)\npredictions","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:38.200599Z","iopub.execute_input":"2024-05-14T18:02:38.201003Z","iopub.status.idle":"2024-05-14T18:02:39.375206Z","shell.execute_reply.started":"2024-05-14T18:02:38.200963Z","shell.execute_reply":"2024-05-14T18:02:39.374042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 가장 높은 확률을 가진 클래스의 인덱스를 찾음\npredicted_class = np.argmax(predictions, axis=1)\n\nprint(\"Predicted class:\", predicted_class)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:39.376483Z","iopub.execute_input":"2024-05-14T18:02:39.376785Z","iopub.status.idle":"2024-05-14T18:02:39.383589Z","shell.execute_reply.started":"2024-05-14T18:02:39.376758Z","shell.execute_reply":"2024-05-14T18:02:39.382328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2-5 Submit!","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/2024-smarcle-ks-3-fashion-mnist2/sample_submission.csv')\n\nsubmission['label'] = predicted_class\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:39.385250Z","iopub.execute_input":"2024-05-14T18:02:39.385680Z","iopub.status.idle":"2024-05-14T18:02:39.418423Z","shell.execute_reply.started":"2024-05-14T18:02:39.385640Z","shell.execute_reply":"2024-05-14T18:02:39.417501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:02:39.419670Z","iopub.execute_input":"2024-05-14T18:02:39.419985Z","iopub.status.idle":"2024-05-14T18:02:39.430737Z","shell.execute_reply.started":"2024-05-14T18:02:39.419957Z","shell.execute_reply":"2024-05-14T18:02:39.429318Z"},"trusted":true},"execution_count":null,"outputs":[]}]}