{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nfrom tqdm import tqdm\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-04T11:51:05.563225Z","iopub.execute_input":"2022-02-04T11:51:05.563589Z","iopub.status.idle":"2022-02-04T11:51:05.57819Z","shell.execute_reply.started":"2022-02-04T11:51:05.563494Z","shell.execute_reply":"2022-02-04T11:51:05.577404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Load data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:51:05.579389Z","iopub.execute_input":"2022-02-04T11:51:05.58Z","iopub.status.idle":"2022-02-04T11:51:09.183832Z","shell.execute_reply.started":"2022-02-04T11:51:05.579949Z","shell.execute_reply":"2022-02-04T11:51:09.183097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### makedict","metadata":{}},{"cell_type":"code","source":"labeldict = dict(zip(train.index,train[\"label\"]))","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:51:09.193483Z","iopub.execute_input":"2022-02-04T11:51:09.193892Z","iopub.status.idle":"2022-02-04T11:51:09.216238Z","shell.execute_reply.started":"2022-02-04T11:51:09.193856Z","shell.execute_reply":"2022-02-04T11:51:09.215245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.iloc[:,1:]\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:51:09.218797Z","iopub.execute_input":"2022-02-04T11:51:09.219106Z","iopub.status.idle":"2022-02-04T11:51:09.229661Z","shell.execute_reply.started":"2022-02-04T11:51:09.219071Z","shell.execute_reply":"2022-02-04T11:51:09.228933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"../input/digit-recognizer/test.csv\")\ntest","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Normarization","metadata":{}},{"cell_type":"code","source":"train = [s/np.linalg.norm(s) for s in tqdm(train.values)]","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:51:09.231644Z","iopub.execute_input":"2022-02-04T11:51:09.232225Z","iopub.status.idle":"2022-02-04T11:51:10.150087Z","shell.execute_reply.started":"2022-02-04T11:51:09.232176Z","shell.execute_reply":"2022-02-04T11:51:10.149048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = [s/np.linalg.norm(s) for s in tqdm(test.values)]","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:51:12.345815Z","iopub.execute_input":"2022-02-04T11:51:12.346647Z","iopub.status.idle":"2022-02-04T11:51:12.926685Z","shell.execute_reply.started":"2022-02-04T11:51:12.346584Z","shell.execute_reply":"2022-02-04T11:51:12.925635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:51:12.92808Z","iopub.execute_input":"2022-02-04T11:51:12.928339Z","iopub.status.idle":"2022-02-04T11:51:12.932759Z","shell.execute_reply.started":"2022-02-04T11:51:12.928305Z","shell.execute_reply":"2022-02-04T11:51:12.931788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Calculate cosine similarity","metadata":{}},{"cell_type":"code","source":"%%time\n\nalldf = pd.DataFrame()\nallres = []\n\n# by reducing memory, divide 4 chunks\n\nfor a in tqdm(range(4)):\n    train2 = train[10500*a:10500*(a+1)]\n    cossim = np.matmul(train2,np.transpose(test))\n    cossim = [s.astype(\"float16\") for s in cossim]\n    allres.append(cossim)\n    \n    del train2,cossim\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:51:12.980737Z","iopub.execute_input":"2022-02-04T11:51:12.980977Z","iopub.status.idle":"2022-02-04T11:51:53.627338Z","shell.execute_reply.started":"2022-02-04T11:51:12.980947Z","shell.execute_reply":"2022-02-04T11:51:53.626248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allres = np.concatenate(allres)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:51:58.606344Z","iopub.execute_input":"2022-02-04T11:51:58.607549Z","iopub.status.idle":"2022-02-04T11:52:01.445697Z","shell.execute_reply.started":"2022-02-04T11:51:58.607496Z","shell.execute_reply":"2022-02-04T11:52:01.444685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alldf = pd.DataFrame(allres)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:52:03.364286Z","iopub.execute_input":"2022-02-04T11:52:03.364727Z","iopub.status.idle":"2022-02-04T11:52:03.369532Z","shell.execute_reply.started":"2022-02-04T11:52:03.364676Z","shell.execute_reply":"2022-02-04T11:52:03.368816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alldf = alldf.T","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:52:28.007569Z","iopub.execute_input":"2022-02-04T11:52:28.008199Z","iopub.status.idle":"2022-02-04T11:52:28.019405Z","shell.execute_reply.started":"2022-02-04T11:52:28.008139Z","shell.execute_reply":"2022-02-04T11:52:28.018385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Confirm cosine similarity result","metadata":{}},{"cell_type":"code","source":"alldf","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:52:31.807024Z","iopub.execute_input":"2022-02-04T11:52:31.8074Z","iopub.status.idle":"2022-02-04T11:52:31.845713Z","shell.execute_reply.started":"2022-02-04T11:52:31.807357Z","shell.execute_reply":"2022-02-04T11:52:31.844782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. calculate the most similar train image in each test image","metadata":{}},{"cell_type":"code","source":"simid = [s.argmax() for s in alldf.values]","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:53:01.842617Z","iopub.execute_input":"2022-02-04T11:53:01.84347Z","iopub.status.idle":"2022-02-04T11:53:19.337428Z","shell.execute_reply.started":"2022-02-04T11:53:01.843395Z","shell.execute_reply":"2022-02-04T11:53:19.336428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. submit","metadata":{}},{"cell_type":"code","source":"sample = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\nsample","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:53:19.33967Z","iopub.execute_input":"2022-02-04T11:53:19.340054Z","iopub.status.idle":"2022-02-04T11:53:19.374386Z","shell.execute_reply.started":"2022-02-04T11:53:19.34Z","shell.execute_reply":"2022-02-04T11:53:19.373533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6.1 convert similar train id to label","metadata":{}},{"cell_type":"code","source":"sample[\"Label\"] = [labeldict[s] for s in simid]\nsample","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:53:19.376091Z","iopub.execute_input":"2022-02-04T11:53:19.376348Z","iopub.status.idle":"2022-02-04T11:53:19.423081Z","shell.execute_reply.started":"2022-02-04T11:53:19.376317Z","shell.execute_reply":"2022-02-04T11:53:19.422179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T11:53:19.425177Z","iopub.execute_input":"2022-02-04T11:53:19.425435Z","iopub.status.idle":"2022-02-04T11:53:19.482101Z","shell.execute_reply.started":"2022-02-04T11:53:19.425404Z","shell.execute_reply":"2022-02-04T11:53:19.481164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}