{"cells":[{"metadata":{"id":"j8s4C77jO_Aj","colab_type":"text"},"cell_type":"markdown","source":"this notebook was made to work on google colab"},{"metadata":{"id":"XFRgnD5CX1ja","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"import os\nfrom pydrive.auth import GoogleAuth\nfrom pydrive.drive import GoogleDrive\nfrom google.colab import auth\nfrom oauth2client.client import GoogleCredentials","execution_count":0,"outputs":[]},{"metadata":{"id":"wUtToybQX1fn","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"auth.authenticate_user()\ngauth = GoogleAuth()\ngauth.credentials = GoogleCredentials.get_application_default()\ndrive = GoogleDrive(gauth)","execution_count":0,"outputs":[]},{"metadata":{"id":"GYuAgoyKYXGK","colab_type":"text"},"cell_type":"markdown","source":"https://drive.google.com/open?id=1VKmIegctrMDkVEhKsr_7m9I5ltMIZ3e3"},{"metadata":{"id":"d7iv5WTHX1dc","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"download = drive.CreateFile({'id': '1VKmIegctrMDkVEhKsr_7m9I5ltMIZ3e3'})\ndownload.GetContentFile('coins.zip')","execution_count":0,"outputs":[]},{"metadata":{"id":"yPs5OrMyltzx","colab_type":"text"},"cell_type":"markdown","source":"https://drive.google.com/open?id=1SDd6umHK0IqwPuPUYs1p0woRI7lsHnvK"},{"metadata":{"id":"1Yejnf_QlJMF","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"download = drive.CreateFile({'id': '1SDd6umHK0IqwPuPUYs1p0woRI7lsHnvK'})\ndownload.GetContentFile('test.zip')","execution_count":0,"outputs":[]},{"metadata":{"id":"bLXoRvhnO2sB","colab_type":"text"},"cell_type":"markdown","source":""},{"metadata":{"id":"wRGbQFPyl6WQ","colab_type":"code","outputId":"f9a622b1-b922-4842-dba2-8d2d7d47784b","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":false},"cell_type":"code","source":"!unzip test.zip -d test","execution_count":0,"outputs":[]},{"metadata":{"id":"6sO1m5ZsX1aD","colab_type":"code","outputId":"37453ee8-e080-44c1-df62-f2a89659c167","colab":{"base_uri":"https://localhost:8080/","height":35},"trusted":false},"cell_type":"code","source":"!pwd","execution_count":0,"outputs":[]},{"metadata":{"id":"woJIQUmzX1XE","colab_type":"code","outputId":"4111550b-f73a-4358-8351-f8e908c4ca97","colab":{"base_uri":"https://localhost:8080/","height":35},"trusted":false},"cell_type":"code","source":"!ls","execution_count":0,"outputs":[]},{"metadata":{"id":"yYAKBg7vX1VE","colab_type":"code","outputId":"dcbf07a4-036a-4759-8e29-4a3f503d74d6","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":false},"cell_type":"code","source":"!unzip coins.zip -d data","execution_count":0,"outputs":[]},{"metadata":{"id":"IAzdXOaOX1SS","colab_type":"code","outputId":"c3e8edde-ff67-4adf-9e7e-fbd03493ddc9","colab":{"base_uri":"https://localhost:8080/","height":35},"trusted":false},"cell_type":"code","source":"!ls","execution_count":0,"outputs":[]},{"metadata":{"id":"WSpyJGOmX1Pl","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"YmQRT740X1My","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"ZdF92zvkcv0w","colab_type":"text"},"cell_type":"markdown","source":"## importing datset and fastai"},{"metadata":{"id":"zdabalONX0dL","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"mAqQVCOGX0Zo","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"-yqB8AqlX0W5","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"Z7kTMcECcv0z","colab_type":"code","colab":{}},"cell_type":"code","source":"import pathlib\nfrom fastai.vision import *\nfrom fastai.metrics import error_rate , accuracy","execution_count":0,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"id":"I9yvRC9ncv07","colab_type":"code","colab":{}},"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport os\n","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"esvWMkT8cv1D","colab_type":"code","outputId":"f39233db-d2b1-4ebe-deb8-4b08e26bd6a7","colab":{"base_uri":"https://localhost:8080/","height":55}},"cell_type":"code","source":"print(os.listdir(\"/content/data/Coins\"))","execution_count":0,"outputs":[]},{"metadata":{"id":"eljS-WSrcv1K","colab_type":"text"},"cell_type":"markdown","source":"## splitting the dataset into test train and validation "},{"metadata":{"trusted":true,"id":"It-K2DGFcv1M","colab_type":"code","outputId":"798d1566-7366-4cbd-8599-86f802dbd93f","colab":{"base_uri":"https://localhost:8080/","height":90}},"cell_type":"code","source":"!pip install split-folders\n","execution_count":0,"outputs":[]},{"metadata":{"id":"3CjH2PCjaff4","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"!mkdir babes","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"WWsV5Hi5cv1R","colab_type":"code","outputId":"8701dc6e-6b5e-4e48-a6e9-ed0504bdff46","colab":{"base_uri":"https://localhost:8080/","height":35}},"cell_type":"code","source":"import split_folders\n\n# Split with a ratio.\n# To only split into training and validation set, set a tuple to `ratio`, i.e, `(.8, .2)`.\nsplit_folders.ratio('/content/data/Coins', output=\"/content/newdata\", seed=1337, ratio=(.8, .1, .1)) # default values","execution_count":0,"outputs":[]},{"metadata":{"id":"M0jMZoL6c15J","colab_type":"code","outputId":"b4b8b02e-c424-4773-b412-30bd6d96ed04","colab":{"base_uri":"https://localhost:8080/","height":55},"trusted":false},"cell_type":"code","source":"print(os.listdir(\"/content/data/Coins\"))","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"9ElTuQjCcv1c","colab_type":"code","outputId":"cc4f356d-c365-4dee-b7c2-8d462d233a0b","colab":{"base_uri":"https://localhost:8080/","height":35}},"cell_type":"code","source":"\n!mv /content/newdata/val /content/newdata/valid \nprint(os.listdir(\"../content/newdata\"))","execution_count":0,"outputs":[]},{"metadata":{"id":"1jgXzWP4dWZ9","colab_type":"text"},"cell_type":"markdown","source":""},{"metadata":{"id":"ft3eIrfifJFh","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"rm -rf /content/newdata/train/*","execution_count":0,"outputs":[]},{"metadata":{"id":"eOB4CtbI2fkF","colab_type":"code","outputId":"aedc00db-7d72-4db1-a47d-a416e1a2706a","colab":{"base_uri":"https://localhost:8080/","height":35},"trusted":false},"cell_type":"code","source":"print(os.listdir(\"../content/newdata/train\"))","execution_count":0,"outputs":[]},{"metadata":{"id":"7GRjZAExbV46","colab_type":"code","outputId":"b6e0b281-0490-4caa-a637-4777912df9a3","colab":{"base_uri":"https://localhost:8080/","height":598},"trusted":false},"cell_type":"code","source":"mv  -v /content/data/Coins/* /content/newdata/train/","execution_count":0,"outputs":[]},{"metadata":{"id":"0DzKMzn8d9jE","colab_type":"code","outputId":"b9f0408b-dae9-4a3e-aaa9-025e481b32ac","colab":{"base_uri":"https://localhost:8080/","height":55},"trusted":false},"cell_type":"code","source":"print(os.listdir(\"../content/newdata/train/taiwan\"))","execution_count":0,"outputs":[]},{"metadata":{"id":"KW9jzUqpcv1m","colab_type":"text"},"cell_type":"markdown","source":"## Feeding the data to fastai "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"id":"CL-_h_FQcv1o","colab_type":"code","colab":{}},"cell_type":"code","source":"from pathlib import Path\npath = Path(\"../content/newdata\")","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"ME7UJN6qcv1t","colab_type":"code","colab":{}},"cell_type":"code","source":"#creating an object of get_transform class from fastai.vision\n#transformations are used for data augmentation\n#since we have a lot of photos in test data which are inverted and which are taken from different angles \n#we need to augment our data \ntfms = get_transforms(do_flip=False)\n#we have normalize our data with google imagenet data statistcs\ndata = ImageDataBunch.from_folder(path,ds_tfms=tfms, size=256).normalize(imagenet_stats)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"CCHPAToLcv1x","colab_type":"code","outputId":"4f687f06-34ec-48df-b9ee-41a35a565378","colab":{"base_uri":"https://localhost:8080/","height":441}},"cell_type":"code","source":"data.show_batch(rows=3, figsize=(7,6))","execution_count":0,"outputs":[]},{"metadata":{"id":"bidoAbhNcv13","colab_type":"text"},"cell_type":"markdown","source":"## Deploying resnet50 CNN model"},{"metadata":{"id":"VoEAnHKkcv14","colab_type":"text"},"cell_type":"markdown","source":"### find the best lr using fastai lr finder [](http://)"},{"metadata":{"trusted":true,"id":"nFixbIWccv15","colab_type":"code","outputId":"4e48657f-971a-4c71-92e4-0271a8e84ba1","colab":{"base_uri":"https://localhost:8080/","height":415}},"cell_type":"code","source":"learn = cnn_learner(data, models.resnet50, metrics=accuracy,callback_fns=ShowGraph)\nlearn.lr_find()\nlearn.recorder.plot()\n","execution_count":0,"outputs":[]},{"metadata":{"id":"AWVGRVzycv1-","colab_type":"text"},"cell_type":"markdown","source":"### the lr range for which the graph has steepest -ve slope is best suited for training"},{"metadata":{"id":"c3kNOiLff-ue","colab_type":"code","outputId":"9af894b0-0bfd-41bf-a93f-1a8a5771bcf8","colab":{"base_uri":"https://localhost:8080/","height":319},"trusted":false},"cell_type":"code","source":"learn.recorder.plot(suggestion=True)\nbest_clf_lr = learn.recorder.min_grad_lr\nlr=best_clf_lr","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"YNdQ1eB8cv1-","colab_type":"code","colab":{}},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"YSXFD2vlcv2C","colab_type":"code","colab":{}},"cell_type":"code","source":"plt.style.use(\"seaborn\")","execution_count":0,"outputs":[]},{"metadata":{"id":"MgQXw3Flcv2H","colab_type":"text"},"cell_type":"markdown","source":"training"},{"metadata":{"trusted":true,"id":"9LGCt6V0cv2J","colab_type":"code","outputId":"08e4bf91-1b48-43d6-881a-9d825bd5015f","colab":{"base_uri":"https://localhost:8080/","height":611}},"cell_type":"code","source":"learn.fit_one_cycle(10, max_lr=0.005248074602497722)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"IqEZB8Oycv2M","colab_type":"code","colab":{}},"cell_type":"code","source":"\nlearn.save('stage-1')","execution_count":0,"outputs":[]},{"metadata":{"id":"fGLJlS3r86Fe","colab_type":"code","outputId":"4cabd278-0af5-4e7c-fe46-dfa37210c4a8","colab":{"base_uri":"https://localhost:8080/","height":322},"trusted":false},"cell_type":"code","source":"learn.load('stage-1')","execution_count":0,"outputs":[]},{"metadata":{"id":"DgawYIaW86AO","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]},{"metadata":{"id":"wtD1Wl3toIzH","colab_type":"code","outputId":"bada42e6-a741-4f97-ffd6-b9618ebf8a30","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":false},"cell_type":"code","source":"learn.fit_one_cycle(10, max_lr=0.005248074602497722)","execution_count":0,"outputs":[]},{"metadata":{"id":"KZAqKXpwcv2R","colab_type":"text"},"cell_type":"markdown","source":"# We are intentionally Overfitting since our data is complete and our test data is very much similiar to train data we can overfit our model to reduce the train loss.\n"},{"metadata":{"id":"baNsktkPcv2S","colab_type":"text"},"cell_type":"markdown","source":"## Investigating top losses"},{"metadata":{"trusted":true,"id":"fmhZsTwkcv2T","colab_type":"code","colab":{}},"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\n\nlosses,idxs = interp.top_losses()\n\nlen(data.valid_ds)==len(losses)==len(idxs)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"nng4WdxPcv2Y","colab_type":"code","colab":{}},"cell_type":"code","source":"interp.plot_top_losses(9, figsize=(15,11))","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"UXQlFzaWcv2d","colab_type":"code","colab":{}},"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\n\nlosses,idxs = interp.top_losses()\n\nlen(data.valid_ds)==len(losses)==len(idxs)","execution_count":0,"outputs":[]},{"metadata":{"id":"kwy6Sj-5cv2i","colab_type":"text"},"cell_type":"markdown","source":"## Making Labels"},{"metadata":{"id":"v6T3d0rycv2k","colab_type":"text"},"cell_type":"markdown","source":"### right now our model is only trained to predict the respective class the classes are the folder no.s. So therfore we need to get labels and assign each label to its respective country. we have used glob library for our task."},{"metadata":{"trusted":true,"id":"PTPlUqG2cv2m","colab_type":"code","colab":{}},"cell_type":"code","source":"p = Path(\"../working/data/train\")","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"wTmqmi-Hcv2p","colab_type":"code","colab":{}},"cell_type":"code","source":"directories = p.glob(\"*\")\n#we have made a list of all the directories present at path p which is the path to our training folder\nlabels=0\n#instantiating the country dictionary.\ncountry_dic={}\n\nfor fol in directories:\n    class_name =int(str(fol).split(\"/\")[-1]) # getting class name from folder name\n    # \n    for img_path in fol.glob(\"*.jpg\"): \n        #getting country name from image name /\n        #some insane string slicing and regex\n        country = \"_\".join(str(img_path).split(\"/\")[-1][8:].split(\".\")[0].split(\"_\")[1:])\n        if country == \"denmark\":\n            labels+=1\n        if class_name not in country_dic.keys():\n            country_dic[class_name] = country","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"ogwz5UV6cv2t","colab_type":"code","colab":{}},"cell_type":"code","source":"labels","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"0BavrvNkcv2v","colab_type":"code","colab":{}},"cell_type":"code","source":"country_dic","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"vU21xOt9cv2z","colab_type":"code","colab":{}},"cell_type":"code","source":"from pathlib import Path\np = Path(\"/content/test/test\")","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"b99yxo5Wcv22","colab_type":"code","colab":{}},"cell_type":"code","source":"dire = p.glob(\"*.jpg\")\noutput = [0 for i in range(1056)]\nfor img_path in dire:\n    img = open_image(img_path)\n    #opening each image in test folder and predicting classes\n    pred_class,pred_idx,outputs = learn.predict(img)\n    #changing predicted classes to predicted country with help of country dic we made above\n    \n    #print(str(img_path).spli/\")[-1].split(\".\")[0])\n    #getting image index from its path and storing predicted country at that index\n    output[int(str(img_path).split(\"/\")[-1].split(\".\")[0])] = pred_class\n    ","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"oixSEg6mcv25","colab_type":"code","colab":{}},"cell_type":"code","source":"output.remove(0)\n# since zeroth index not needed","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"aBUHbKIEcv2-","colab_type":"code","colab":{}},"cell_type":"code","source":"output = np.array(output,dtype=\"str\")","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"plufreCucv3A","colab_type":"code","colab":{}},"cell_type":"code","source":"df = pd.DataFrame(output)\ndf.to_csv(\"../content/Answer.csv\")","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"rr7HNh7Gcv3D","colab_type":"code","colab":{}},"cell_type":"code","source":"df.head()","execution_count":0,"outputs":[]},{"metadata":{"id":"SeH8M7_Lcv3H","colab_type":"text"},"cell_type":"markdown","source":"## Downloading the answer"},{"metadata":{"trusted":true,"id":"RLdJRTvwcv3H","colab_type":"code","colab":{}},"cell_type":"code","source":"import base64\nfrom IPython.display import HTML\n# function that takes in a dataframe and creates a text link to  \n# download it (will only work for files < 2MB or so)\ndef create_download_link(df, title = \"Download CSV file\", filename = \"Answer.csv\"):  \n    csv = df.to_csv()\n    #df.index = (1,np.arange(len(df)+1))\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\n\n\n# create a link to download the dataframe\ncreate_download_link(df)","execution_count":0,"outputs":[]},{"metadata":{"id":"DJsrp8gOnG_-","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"colab":{"name":"kernel708bf0ee15.ipynb","provenance":[],"machine_shape":"hm"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":1}