{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"We can create short to do list what we will do in that note book\n<br>\n\n\n* Data Handling Section\n  * Data Missing Value Check\n  * Data Distrubation Fix\n    * Target Data Distribution\n    * Data Augmentation\n  * Dataset Shape Transformation  \n    * Data Preparation for Model\n    * Data Resolution Update\n\n* Machine Learning Section\n  * Introduction LET-5 Model\n  * Basis Model Creation\n  * Improvment Section\n\n* Conculution\n","metadata":{}},{"cell_type":"code","source":"# Tensor-Flow Import Section\nimport tensorflow as tf\nfrom tensorflow.keras.layers import RandomRotation\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, Dropout\n\n# Old friend Sk-Learn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\n# Statistical libraries\nimport pandas as pd\nfrom scipy.ndimage import zoom\nimport numpy as np\nfrom random import randint\n\n# Visulation pack\nfrom matplotlib import pyplot as plt\nimport plotly\nimport plotly.graph_objects as go\nimport plotly.express as px\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-01T15:19:24.083247Z","iopub.execute_input":"2023-10-01T15:19:24.083781Z","iopub.status.idle":"2023-10-01T15:19:24.094465Z","shell.execute_reply.started":"2023-10-01T15:19:24.083736Z","shell.execute_reply":"2023-10-01T15:19:24.093174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Import Section","metadata":{}},{"cell_type":"code","source":"data_train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:19:24.096685Z","iopub.execute_input":"2023-10-01T15:19:24.097754Z","iopub.status.idle":"2023-10-01T15:19:26.095497Z","shell.execute_reply.started":"2023-10-01T15:19:24.097661Z","shell.execute_reply":"2023-10-01T15:19:26.094574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Handling Section\n\nMost of the things we will do is simple and standart things for Convulation Neural Network. Beside these standart procecure we have some interesting thing.\n","metadata":{}},{"cell_type":"code","source":"data_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:19:26.096963Z","iopub.execute_input":"2023-10-01T15:19:26.097935Z","iopub.status.idle":"2023-10-01T15:19:26.125589Z","shell.execute_reply.started":"2023-10-01T15:19:26.097901Z","shell.execute_reply":"2023-10-01T15:19:26.124333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Missing Value Check","metadata":{}},{"cell_type":"code","source":"sum_nullVal = data_train.isnull().sum().sum()\n\nprint(f\"Number of null values in data set : {sum_nullVal}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:19:26.127190Z","iopub.execute_input":"2023-10-01T15:19:26.127825Z","iopub.status.idle":"2023-10-01T15:19:26.159395Z","shell.execute_reply.started":"2023-10-01T15:19:26.127790Z","shell.execute_reply":"2023-10-01T15:19:26.158432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Distrubation Fix","metadata":{}},{"cell_type":"code","source":"graph = plt.hist(data_train.label, rwidth= 0.6 )","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:19:26.162222Z","iopub.execute_input":"2023-10-01T15:19:26.162500Z","iopub.status.idle":"2023-10-01T15:19:26.383520Z","shell.execute_reply.started":"2023-10-01T15:19:26.162462Z","shell.execute_reply":"2023-10-01T15:19:26.382613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The thing about that distribution is being not even. On the ideal case our target data should be even to prevent the learning error in our model. <br>\nTo achive that in our non-even image data set we use the \" Data Augmentation \" technique.","metadata":{}},{"cell_type":"code","source":"# Begining variable initilation\n# target_count = data_train.label.value_counts().values[0]\ntarget_count = 6000\nlist_unique = data_train.label.unique()\nsub_set = pd.DataFrame(columns = data_train.columns)\nfor val in list_unique :\n\n  temp_dataset = data_train.loc[data_train.label == val].drop('label', axis=1)\n  len_temp_dataset = len(temp_dataset)\n  random_stop_point = len(temp_dataset) - 1\n\n  while len_temp_dataset < target_count :\n\n      random_index = randint(0,random_stop_point)\n\n      temp_data_array = temp_dataset.iloc[random_index].values\n      temp_data_tensor = tf.constant(temp_data_array.reshape(28, 28, 1))\n      gnrtd_data_tensor = RandomRotation( factor= (-0.05, 0.05) )(temp_data_tensor).numpy()\n\n      gnrtd_data_list = list(gnrtd_data_tensor.reshape(1,784).astype(int)[0])\n      gnrtd_data_list.insert(0, val)\n\n      sub_set.loc[len(sub_set)] = gnrtd_data_list\n\n      len_temp_dataset += 1","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:19:26.384805Z","iopub.execute_input":"2023-10-01T15:19:26.385635Z","iopub.status.idle":"2023-10-01T15:28:42.130031Z","shell.execute_reply.started":"2023-10-01T15:19:26.385604Z","shell.execute_reply":"2023-10-01T15:28:42.128973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = pd.concat([data_train, sub_set],ignore_index= True)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:28:42.131555Z","iopub.execute_input":"2023-10-01T15:28:42.132287Z","iopub.status.idle":"2023-10-01T15:28:42.258383Z","shell.execute_reply.started":"2023-10-01T15:28:42.132251Z","shell.execute_reply":"2023-10-01T15:28:42.257425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph = plt.hist(data_train.label, rwidth= 0.6 )","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:28:42.259517Z","iopub.execute_input":"2023-10-01T15:28:42.259848Z","iopub.status.idle":"2023-10-01T15:28:42.856006Z","shell.execute_reply.started":"2023-10-01T15:28:42.259817Z","shell.execute_reply":"2023-10-01T15:28:42.854954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset Shape Transformation","metadata":{}},{"cell_type":"markdown","source":"**Data Preparation for Model**","metadata":{}},{"cell_type":"code","source":"data_train_input = data_train.drop('label', axis=1).values.reshape(-1, 28, 28, 1).astype(float)\ndata_train_target = pd.get_dummies(data_train.label, prefix = \"label_\")","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:28:42.857643Z","iopub.execute_input":"2023-10-01T15:28:42.858286Z","iopub.status.idle":"2023-10-01T15:28:43.144508Z","shell.execute_reply.started":"2023-10-01T15:28:42.858240Z","shell.execute_reply":"2023-10-01T15:28:43.143565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Resolution Update**","metadata":{}},{"cell_type":"code","source":"# Define the scale factor (for example, double the size)\nscale_factor = 1.14\n\n# Zoom the image using NumPy\ndata_train_input = zoom(data_train_input, (1, scale_factor, scale_factor, 1), order=1)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:28:43.145613Z","iopub.execute_input":"2023-10-01T15:28:43.145961Z","iopub.status.idle":"2023-10-01T15:28:58.212491Z","shell.execute_reply.started":"2023-10-01T15:28:43.145931Z","shell.execute_reply":"2023-10-01T15:28:58.211539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_input, test_input, train_target, test_target = train_test_split(data_train_input, data_train_target, test_size= 0.2, random_state= 19)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:28:58.214211Z","iopub.execute_input":"2023-10-01T15:28:58.214551Z","iopub.status.idle":"2023-10-01T15:28:58.374320Z","shell.execute_reply.started":"2023-10-01T15:28:58.214522Z","shell.execute_reply":"2023-10-01T15:28:58.373361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  Machine Learning Section\n\nAs you can guess we are doing build our model and it with data sets we create int the last part.","metadata":{}},{"cell_type":"markdown","source":"### Introduction LET-5 Model","metadata":{}},{"cell_type":"markdown","source":"### Basis Model Creation","metadata":{}},{"cell_type":"markdown","source":"**Let-5 Model Build Stage**","metadata":{}},{"cell_type":"code","source":"Let5_Model = tf.keras.Sequential()\n\nLet5_Model.add(\n    Conv2D(\n      filters = 32,\n      kernel_size = 5,\n      strides = 1,\n      activation = 'relu',\n      )\n    )\n\nLet5_Model.add(\n  MaxPooling2D(\n    pool_size = (2,2),\n    strides = 2,\n    )\n  )\n\nLet5_Model.add(\n    Conv2D(\n      filters = 64,\n      kernel_size = 5,\n      strides = 1,\n      activation = 'relu'\n      )\n    )\n\nLet5_Model.add(\n  MaxPooling2D(\n    pool_size = (2,2),\n    strides = 2,\n    )\n  )\n\nLet5_Model.add(\n  Flatten()\n  )\n\nLet5_Model.add(\n  Dense(\n    units = 256,\n    activation = 'relu',\n    )\n  )\n\"\"\"\nLet5_Model.add(\n  Dense(\n    units = 120,\n    activation = 'relu',\n    )\n  )\n\"\"\"\nLet5_Model.add(\n  Dense(\n    units = 84,\n    activation = 'relu',\n    )\n  )\n\nLet5_Model.add(\n    Dense(\n      units = 10,\n      activation= 'softmax'\n    )\n)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:28:58.375865Z","iopub.execute_input":"2023-10-01T15:28:58.376238Z","iopub.status.idle":"2023-10-01T15:28:58.396614Z","shell.execute_reply.started":"2023-10-01T15:28:58.376205Z","shell.execute_reply":"2023-10-01T15:28:58.395697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model Optimizer Initializing**","metadata":{"execution":{"iopub.status.busy":"2023-10-01T14:47:03.194182Z","iopub.execute_input":"2023-10-01T14:47:03.195201Z","iopub.status.idle":"2023-10-01T14:47:03.202653Z","shell.execute_reply.started":"2023-10-01T14:47:03.195177Z","shell.execute_reply":"2023-10-01T14:47:03.200839Z"}}},{"cell_type":"code","source":"model_opt = tf.optimizers.legacy.RMSprop(learning_rate=0.001, rho=0.9, epsilon=1e-09, decay=0.0)\n\n# Compile the model\n\nLet5_Model.compile(optimizer = model_opt , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:28:58.397819Z","iopub.execute_input":"2023-10-01T15:28:58.398132Z","iopub.status.idle":"2023-10-01T15:28:58.407766Z","shell.execute_reply.started":"2023-10-01T15:28:58.398103Z","shell.execute_reply":"2023-10-01T15:28:58.406722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model Train & Test Stage**\n\n---\n","metadata":{}},{"cell_type":"code","source":"# Train the our model as we can understand\nLet5_Model.fit(train_input, train_target, batch_size=160, epochs= 25)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:28:58.411392Z","iopub.execute_input":"2023-10-01T15:28:58.411608Z","iopub.status.idle":"2023-10-01T15:29:37.548359Z","shell.execute_reply.started":"2023-10-01T15:28:58.411589Z","shell.execute_reply":"2023-10-01T15:29:37.547385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now we can mesure how good we are\nLet5_model_pred = Let5_Model.predict(test_input)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:29:37.549900Z","iopub.execute_input":"2023-10-01T15:29:37.550222Z","iopub.status.idle":"2023-10-01T15:29:38.980132Z","shell.execute_reply.started":"2023-10-01T15:29:37.550192Z","shell.execute_reply":"2023-10-01T15:29:38.979229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error_val = mean_squared_error(test_target,Let5_model_pred)\nprint(f\"The error rate of the our basis Let-5 model is : {error_val}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:44:59.053076Z","iopub.execute_input":"2023-10-01T15:44:59.053428Z","iopub.status.idle":"2023-10-01T15:44:59.063164Z","shell.execute_reply.started":"2023-10-01T15:44:59.053401Z","shell.execute_reply":"2023-10-01T15:44:59.061817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"The accurcy of the our basis Let-5 model is : {1- error_val}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:44:59.911387Z","iopub.execute_input":"2023-10-01T15:44:59.912028Z","iopub.status.idle":"2023-10-01T15:44:59.917569Z","shell.execute_reply.started":"2023-10-01T15:44:59.911996Z","shell.execute_reply":"2023-10-01T15:44:59.916518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Transform Matrix**\n\nThat is the basic transformation matrix to transform our output matrix into what competation want as a answer.\n","metadata":{}},{"cell_type":"code","source":"transform_matrix = np.array(range(10)).reshape(10,1)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:29:39.002726Z","iopub.execute_input":"2023-10-01T15:29:39.003415Z","iopub.status.idle":"2023-10-01T15:29:39.009010Z","shell.execute_reply.started":"2023-10-01T15:29:39.003383Z","shell.execute_reply":"2023-10-01T15:29:39.007941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"At that point we create the our basis model and train with our data. In the math we reach the %99.7 accurcy on our model. However, we know we have a room to improvement. Because of that we are going to push our model beyond the what it can already do.","metadata":{}},{"cell_type":"code","source":"pred_matrix = np.dot(Let5_model_pred,transform_matrix)\ntarget_matrix = np.dot(test_target.values,transform_matrix).astype(float)\n\nfilter_matrix = pred_matrix == target_matrix\n\npred_matrix = pred_matrix[~filter_matrix]\ntarget_matrix = target_matrix[~filter_matrix]\n\npred_matrix = pred_matrix.reshape(1,pred_matrix.shape[0])[0]\ntarget_matrix = target_matrix.reshape(1, target_matrix.shape[0])[0]\n\nheat_matrix = np.zeros((10,10))\n\nfor index_ in range(len(pred_matrix)) :\n\n  index_one = int(pred_matrix[index_])\n  index_two = int(target_matrix[index_])\n\n  heat_matrix[index_one][index_two] = heat_matrix[index_one][index_two]+  1\n\nheat_matrix = heat_matrix.astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:29:39.010551Z","iopub.execute_input":"2023-10-01T15:29:39.011372Z","iopub.status.idle":"2023-10-01T15:29:39.028194Z","shell.execute_reply.started":"2023-10-01T15:29:39.011341Z","shell.execute_reply":"2023-10-01T15:29:39.026866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure(\n        data=go.Heatmap(\n          z = heat_matrix,\n          text = heat_matrix,\n          texttemplate=\"%{text}\",\n          textfont={\"size\":8},\n          colorscale = \"gray\"\n          )\n        )\n\nfig.update_xaxes(title_text = \"Actual Value\")\nfig.update_yaxes(title_text = \"Prediction Value\")\n\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:29:39.033329Z","iopub.execute_input":"2023-10-01T15:29:39.033853Z","iopub.status.idle":"2023-10-01T15:29:39.067542Z","shell.execute_reply.started":"2023-10-01T15:29:39.033810Z","shell.execute_reply":"2023-10-01T15:29:39.066342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"----","metadata":{}},{"cell_type":"markdown","source":"**Prediction For the Test Data**","metadata":{}},{"cell_type":"code","source":"data_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:29:39.071289Z","iopub.execute_input":"2023-10-01T15:29:39.074885Z","iopub.status.idle":"2023-10-01T15:29:40.086996Z","shell.execute_reply.started":"2023-10-01T15:29:39.074843Z","shell.execute_reply":"2023-10-01T15:29:40.085961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:29:40.088312Z","iopub.execute_input":"2023-10-01T15:29:40.088631Z","iopub.status.idle":"2023-10-01T15:29:40.119104Z","shell.execute_reply.started":"2023-10-01T15:29:40.088601Z","shell.execute_reply":"2023-10-01T15:29:40.118194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:29:40.120366Z","iopub.execute_input":"2023-10-01T15:29:40.120865Z","iopub.status.idle":"2023-10-01T15:29:40.136528Z","shell.execute_reply.started":"2023-10-01T15:29:40.120836Z","shell.execute_reply":"2023-10-01T15:29:40.135551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test = data_test.values.reshape(-1, 28, 28, 1)\n\n# Define the scale factor (for example, double the size)\nscale_factor = 1.14\n\n# Zoom the image using NumPy\ndata_test = zoom(data_test, (1, scale_factor, scale_factor, 1), order=1)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:29:40.137640Z","iopub.execute_input":"2023-10-01T15:29:40.138179Z","iopub.status.idle":"2023-10-01T15:29:46.930692Z","shell.execute_reply.started":"2023-10-01T15:29:40.138148Z","shell.execute_reply":"2023-10-01T15:29:46.929719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model prediction\nTest_prediction = Let5_Model.predict(data_test)\nTest_prediction","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:46:28.890516Z","iopub.execute_input":"2023-10-01T15:46:28.890861Z","iopub.status.idle":"2023-10-01T15:46:31.973079Z","shell.execute_reply.started":"2023-10-01T15:46:28.890832Z","shell.execute_reply":"2023-10-01T15:46:31.971948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.dot(np.round(Test_prediction), transform_matrix).astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:46:51.764440Z","iopub.execute_input":"2023-10-01T15:46:51.764806Z","iopub.status.idle":"2023-10-01T15:46:51.776814Z","shell.execute_reply.started":"2023-10-01T15:46:51.764777Z","shell.execute_reply":"2023-10-01T15:46:51.775462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_csv = np.dot(np.round(Test_prediction), transform_matrix).astype(int)\ntest_pred_csv = test_pred_csv.reshape(1,len(test_pred_csv))[0]\ntest_pred_csv","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:46:54.044150Z","iopub.execute_input":"2023-10-01T15:46:54.044523Z","iopub.status.idle":"2023-10-01T15:46:54.061056Z","shell.execute_reply.started":"2023-10-01T15:46:54.044495Z","shell.execute_reply":"2023-10-01T15:46:54.059758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example data as a dictionary of lists\ndata = {'ImageId': range(1,len(test_pred_csv)+1),\n        'Label': test_pred_csv}\n\n# Create a DataFrame\npred_df = pd.DataFrame(data)\n\n# Let's give quick look\npred_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:47:11.534866Z","iopub.execute_input":"2023-10-01T15:47:11.535193Z","iopub.status.idle":"2023-10-01T15:47:11.544229Z","shell.execute_reply.started":"2023-10-01T15:47:11.535169Z","shell.execute_reply":"2023-10-01T15:47:11.543185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df.to_csv('/kaggle/working/phd_stipends.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:47:12.619294Z","iopub.execute_input":"2023-10-01T15:47:12.619616Z","iopub.status.idle":"2023-10-01T15:47:12.653918Z","shell.execute_reply.started":"2023-10-01T15:47:12.619591Z","shell.execute_reply":"2023-10-01T15:47:12.653040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/working/phd_stipends.csv')\ndata","metadata":{"execution":{"iopub.status.busy":"2023-10-01T15:47:13.525879Z","iopub.execute_input":"2023-10-01T15:47:13.526209Z","iopub.status.idle":"2023-10-01T15:47:13.543382Z","shell.execute_reply.started":"2023-10-01T15:47:13.526182Z","shell.execute_reply":"2023-10-01T15:47:13.542312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}