{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](https://storage.googleapis.com/kaggle-competitions/kaggle/36073/logos/header.png?t=2022-04-12-20-53-30)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T18:39:54.474251Z","iopub.execute_input":"2022-04-21T18:39:54.474986Z","iopub.status.idle":"2022-04-21T18:39:55.157888Z","shell.execute_reply.started":"2022-04-21T18:39:54.474943Z","shell.execute_reply":"2022-04-21T18:39:55.157081Z"}}},{"cell_type":"markdown","source":"# The task is to denoise “shabby” pages, reversing the effects imposed on them by Augraphy","metadata":{}},{"cell_type":"markdown","source":"### Importing some basic libraries required to play with the images","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-28T16:29:16.94777Z","iopub.execute_input":"2022-04-28T16:29:16.948095Z","iopub.status.idle":"2022-04-28T16:29:16.953222Z","shell.execute_reply.started":"2022-04-28T16:29:16.948062Z","shell.execute_reply":"2022-04-28T16:29:16.952255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Project Root directory:\nBASE_DIR = \"/kaggle/input/denoising-shabby-pages\"","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:29:24.207124Z","iopub.execute_input":"2022-04-28T16:29:24.207939Z","iopub.status.idle":"2022-04-28T16:29:24.211287Z","shell.execute_reply.started":"2022-04-28T16:29:24.207903Z","shell.execute_reply":"2022-04-28T16:29:24.210599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glob.glob(BASE_DIR+'/*')","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:29:32.247681Z","iopub.execute_input":"2022-04-28T16:29:32.248154Z","iopub.status.idle":"2022-04-28T16:29:32.257168Z","shell.execute_reply.started":"2022-04-28T16:29:32.248115Z","shell.execute_reply":"2022-04-28T16:29:32.256291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_img_paths = sorted(glob.glob(BASE_DIR+\"/train_cleaned/train_cleaned/*.png\"))\ntrain_shabby_paths = sorted(glob.glob(BASE_DIR+\"/train/train/*.png\"))","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:29:58.04779Z","iopub.execute_input":"2022-04-28T16:29:58.048083Z","iopub.status.idle":"2022-04-28T16:29:58.58319Z","shell.execute_reply.started":"2022-04-28T16:29:58.04805Z","shell.execute_reply":"2022-04-28T16:29:58.582387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(cleaned_img_paths),len(train_shabby_paths)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:29:58.91654Z","iopub.execute_input":"2022-04-28T16:29:58.916815Z","iopub.status.idle":"2022-04-28T16:29:58.922411Z","shell.execute_reply.started":"2022-04-28T16:29:58.916789Z","shell.execute_reply":"2022-04-28T16:29:58.921512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Display some of the shabby images given for training","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 5))\nprint(\"Input samples:\")\nfor i,pth in enumerate(train_shabby_paths[0:20]):\n    im = cv2.imread(pth)\n    plt.subplot(2, 10, i+1)\n    plt.imshow(im, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:00.726394Z","iopub.execute_input":"2022-04-28T16:30:00.727131Z","iopub.status.idle":"2022-04-28T16:30:03.219267Z","shell.execute_reply.started":"2022-04-28T16:30:00.727094Z","shell.execute_reply":"2022-04-28T16:30:03.218271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 5))\nprint(\"Target samples:\")\nfor i,pth in enumerate(cleaned_img_paths[0:20]):\n    im = cv2.imread(pth)\n    plt.subplot(2, 10, i+1)\n    plt.imshow(im, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:03.221037Z","iopub.execute_input":"2022-04-28T16:30:03.22129Z","iopub.status.idle":"2022-04-28T16:30:05.588911Z","shell.execute_reply.started":"2022-04-28T16:30:03.221261Z","shell.execute_reply":"2022-04-28T16:30:05.587955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = []\n# for i,pth in enumerate(train_shabby_paths):\n#     im = cv2.imread(pth,0)\n#     X_train.append(im)\n# X_train = np.array(X_train)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:05.590347Z","iopub.execute_input":"2022-04-28T16:30:05.590589Z","iopub.status.idle":"2022-04-28T16:30:05.594911Z","shell.execute_reply.started":"2022-04-28T16:30:05.59056Z","shell.execute_reply":"2022-04-28T16:30:05.593972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = []\n# for i,pth in enumerate(cleaned_img_paths):\n#     im = cv2.imread(pth,0)\n#     y_train.append(im)\n# y_train = np.array(y_train)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:05.597049Z","iopub.execute_input":"2022-04-28T16:30:05.597334Z","iopub.status.idle":"2022-04-28T16:30:05.609465Z","shell.execute_reply.started":"2022-04-28T16:30:05.597301Z","shell.execute_reply":"2022-04-28T16:30:05.608445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train.shape, y_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:05.611511Z","iopub.execute_input":"2022-04-28T16:30:05.611789Z","iopub.status.idle":"2022-04-28T16:30:05.625564Z","shell.execute_reply.started":"2022-04-28T16:30:05.611756Z","shell.execute_reply":"2022-04-28T16:30:05.6244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have 700 shabby pages and corresponding cleaned images with the size of 400x400","metadata":{}},{"cell_type":"markdown","source":"Various approaches are available to denoise shabby images.\n\nFeel free to experiment and find best suitable algo/model","metadata":{}},{"cell_type":"code","source":"#TODO: Do your stuff here","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:06.36604Z","iopub.execute_input":"2022-04-28T16:30:06.366363Z","iopub.status.idle":"2022-04-28T16:30:06.369996Z","shell.execute_reply.started":"2022-04-28T16:30:06.366321Z","shell.execute_reply":"2022-04-28T16:30:06.369368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Result preparation:","metadata":{}},{"cell_type":"markdown","source":"### Once the solution is ready, make the submission with the following steps:","metadata":{}},{"cell_type":"code","source":"TEST_DIR = BASE_DIR+\"/test/test/\"\nOUTPUT_DIR = \"/kaggle/working/results\"\n\ntry:\n    os.mkdir(OUTPUT_DIR)\nexcept Exception:\n    pass","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:07.291581Z","iopub.execute_input":"2022-04-28T16:30:07.29212Z","iopub.status.idle":"2022-04-28T16:30:07.297425Z","shell.execute_reply.started":"2022-04-28T16:30:07.292077Z","shell.execute_reply":"2022-04-28T16:30:07.296405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_img_paths = sorted(glob.glob(TEST_DIR+\"*.png\"))","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:07.546327Z","iopub.execute_input":"2022-04-28T16:30:07.547173Z","iopub.status.idle":"2022-04-28T16:30:07.73075Z","shell.execute_reply.started":"2022-04-28T16:30:07.547129Z","shell.execute_reply":"2022-04-28T16:30:07.72994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(final_img_paths)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:07.746384Z","iopub.execute_input":"2022-04-28T16:30:07.746984Z","iopub.status.idle":"2022-04-28T16:30:07.751902Z","shell.execute_reply.started":"2022-04-28T16:30:07.74694Z","shell.execute_reply":"2022-04-28T16:30:07.751277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We have to predict/recreate 300 cleaned images from the given test set to make submissions","metadata":{}},{"cell_type":"code","source":"for i,full_pth in enumerate(final_img_paths):\n    name = i+1\n    pth = full_pth.split('/')[-1]\n    img = cv2.imread(TEST_DIR+pth.strip(),0)\n    res = img.copy()\n    '''\n        TODO: Predict/recreate cleaned version\n    '''\n    cv2.imwrite(f'{OUTPUT_DIR}/{i}.png',res)","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:08.306428Z","iopub.execute_input":"2022-04-28T16:30:08.306857Z","iopub.status.idle":"2022-04-28T16:30:14.329317Z","shell.execute_reply.started":"2022-04-28T16:30:08.306828Z","shell.execute_reply":"2022-04-28T16:30:14.328268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualize the test set:","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 5))\nfor i,pth in enumerate(final_img_paths[0:20]):\n    im = cv2.imread(pth,0)\n    plt.subplot(2, 10, i+1)\n    plt.imshow(im, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:14.331383Z","iopub.execute_input":"2022-04-28T16:30:14.331724Z","iopub.status.idle":"2022-04-28T16:30:16.511228Z","shell.execute_reply.started":"2022-04-28T16:30:14.331678Z","shell.execute_reply":"2022-04-28T16:30:16.510272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualize some of the results:","metadata":{}},{"cell_type":"code","source":"results_img_paths = glob.glob(f'{OUTPUT_DIR}/*.png')\nplt.figure(figsize=(20, 5))\nprint(\"Results\")\nfor i,pth in enumerate(results_img_paths[0:20]):\n    im = cv2.imread(pth)\n    plt.subplot(2, 10, i+1)\n    plt.imshow(im, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:16.512631Z","iopub.execute_input":"2022-04-28T16:30:16.512919Z","iopub.status.idle":"2022-04-28T16:30:18.846497Z","shell.execute_reply.started":"2022-04-28T16:30:16.512883Z","shell.execute_reply":"2022-04-28T16:30:18.845215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make a submission:","metadata":{}},{"cell_type":"markdown","source":"To create submission.csv file, run the following code block...","metadata":{}},{"cell_type":"code","source":"# Make_submission.py:\nimport random\nimport os\nimport cv2\n\n################################################################################\n# CHANGE THIS\n################################################################################\ncleaned_images_dir = OUTPUT_DIR\n\n################################################################################\n# DON'T CHANGE ANYTHING BELOW HERE\n################################################################################\ndef select_pixels(img):\n    y,x = img.shape\n\n    pixels = list()\n\n    for i in range(10000):\n        pixel = (random.randrange(y), random.randrange(x))\n\n        if pixel not in pixels:\n            pixels.append(pixel)\n\n    return pixels\n\nrandom.seed(0)\ncleaned_images = sorted(os.listdir(cleaned_images_dir))\n\nwith open(\"submission.csv\", \"w\") as submission_file:\n    submission_file.write(\"id,predicted\\n\")\n\n    filenum = 1\n    for image in cleaned_images:\n\n        img = cv2.imread(\"{}/{}\".format(cleaned_images_dir, image), cv2.IMREAD_GRAYSCALE)\n        pixels = select_pixels(img)\n\n        for pixel in pixels:\n            y,x = pixel\n            submission_file.write(\"{}_{}_{},{}\\n\".format(filenum, y, x, 0.99))\n\n        filenum += 1","metadata":{"execution":{"iopub.status.busy":"2022-04-28T16:30:18.848715Z","iopub.execute_input":"2022-04-28T16:30:18.848991Z","iopub.status.idle":"2022-04-28T16:36:40.039733Z","shell.execute_reply.started":"2022-04-28T16:30:18.84895Z","shell.execute_reply":"2022-04-28T16:36:40.038889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The world belongs to those who READ! Let's make shabby pages readable, those are not to be thrown in the trash! :D\n# Thanks!","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}