{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nimport zipfile\nimport os\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-25T15:28:04.878413Z","iopub.execute_input":"2021-06-25T15:28:04.878812Z","iopub.status.idle":"2021-06-25T15:28:05.06323Z","shell.execute_reply.started":"2021-06-25T15:28:04.878772Z","shell.execute_reply":"2021-06-25T15:28:05.062159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# zip file handler  \nzip = zipfile.ZipFile('/kaggle/input/dogs-vs-cats/train.zip')\nzip.extractall('/kaggle/temp/')\n\n# list available files in the container\n#print (zip.namelist())\n'''import shutil\nshutil.rmtree('/kaggle/working/train')'''","metadata":{"execution":{"iopub.status.busy":"2021-06-02T07:41:27.036054Z","iopub.execute_input":"2021-06-02T07:41:27.036373Z","iopub.status.idle":"2021-06-02T07:41:40.098714Z","shell.execute_reply.started":"2021-06-02T07:41:27.036344Z","shell.execute_reply":"2021-06-02T07:41:40.097686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filenames = os.listdir('/kaggle/temp/train') \n'''example: ['dog.890.jpg',\n 'dog.1178.jpg',\n 'dog.7845.jpg',\n 'dog.4632.jpg',\n 'cat.3660.jpg']'''\nimport re\n\ncategory=[]\nid = []\nfor name in filenames:\n    id.append(name.split('.')[1])\n    category.append(name.split('.')[0])\n    '''if re.match(r\"dog.*\", name):\n        category.append(1)\n    else: \n        category.append(0)'''\ndf = pd.DataFrame.from_dict({'id':id,'filename':filenames,'category':category})\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-02T07:41:40.100568Z","iopub.execute_input":"2021-06-02T07:41:40.100913Z","iopub.status.idle":"2021-06-02T07:41:40.191757Z","shell.execute_reply.started":"2021-06-02T07:41:40.100879Z","shell.execute_reply":"2021-06-02T07:41:40.190782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby('category').count()","metadata":{"execution":{"iopub.status.busy":"2021-06-02T07:41:40.193738Z","iopub.execute_input":"2021-06-02T07:41:40.194113Z","iopub.status.idle":"2021-06-02T07:41:40.792949Z","shell.execute_reply.started":"2021-06-02T07:41:40.194082Z","shell.execute_reply":"2021-06-02T07:41:40.791923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain, test = train_test_split(df, test_size=0.3, stratify=df.category)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T07:41:40.794418Z","iopub.execute_input":"2021-06-02T07:41:40.794777Z","iopub.status.idle":"2021-06-02T07:41:42.023232Z","shell.execute_reply.started":"2021-06-02T07:41:40.794747Z","shell.execute_reply":"2021-06-02T07:41:42.022026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('category').count()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby('category').count()","metadata":{"execution":{"iopub.status.busy":"2021-06-02T07:50:06.925616Z","iopub.execute_input":"2021-06-02T07:50:06.926221Z","iopub.status.idle":"2021-06-02T07:50:06.942915Z","shell.execute_reply.started":"2021-06-02T07:50:06.926186Z","shell.execute_reply":"2021-06-02T07:50:06.941932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom keras.preprocessing.image import ImageDataGenerator\n\n'''train_datagen = ImageDataGenerator(rescale = 1./255,\n                                   shear_range = 0.2,\n                                   zoom_range = 0.2,\n                                   horizontal_flip = True)\n\ntraining_set = train_datagen.flow_from_dataframe(train,directory='/kaggle/temp/train',x_col=\"filename\", y_col=\"category\",\n                                                 target_size = (64, 64),\n                                                 batch_size = 32,\n                                                 class_mode = 'binary')\n\n# Preprocessing the Test set\ntest_datagen = ImageDataGenerator(rescale = 1./255)\ntest_set = test_datagen.flow_from_dataframe(test,directory='/kaggle/temp/train',x_col=\"filename\", y_col=\"category\",\n                                            target_size = (64, 64),\n                                            batch_size = 32,\n                                            class_mode = 'binary')'''","metadata":{"execution":{"iopub.status.busy":"2021-06-02T07:52:12.010343Z","iopub.execute_input":"2021-06-02T07:52:12.01069Z","iopub.status.idle":"2021-06-02T07:52:12.343888Z","shell.execute_reply.started":"2021-06-02T07:52:12.010662Z","shell.execute_reply":"2021-06-02T07:52:12.34287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T07:41:48.348094Z","iopub.execute_input":"2021-06-02T07:41:48.348527Z","iopub.status.idle":"2021-06-02T07:41:48.354005Z","shell.execute_reply.started":"2021-06-02T07:41:48.348483Z","shell.execute_reply":"2021-06-02T07:41:48.35295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''# Building the CNN - 84.5% accuracy\nfrom keras.models import Model,Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, Dropout, Dense,Flatten\n# Initialising the CNN\ncnn = tf.keras.models.Sequential()\n\n# Step 1 - Convolution\ncnn.add(tf.keras.layers.Conv2D(filters=32, kernel_size=3, activation='relu', input_shape=[64, 64, 3]))\n\n# Step 2 - Pooling\ncnn.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))\ncnn.add(Dropout(0.2))\n# Adding a second convolutional layer\ncnn.add(tf.keras.layers.Conv2D(filters=32, kernel_size=3, activation='relu'))\ncnn.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))\ncnn.add(tf.keras.layers.Conv2D(filters=64, kernel_size=3, activation='relu'))\ncnn.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))\ncnn.add(Dropout(0.2))\ncnn.add(tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation='relu'))\ncnn.add(tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation='relu'))\ncnn.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))\n\n# Step 3 - Flattening\ncnn.add(tf.keras.layers.Flatten())\n\n# Step 4 - Full Connection\ncnn.add(tf.keras.layers.Dense(units=128, activation='relu'))\ncnn.add(Dropout(0.2))\n# Step 5 - Output Layer\ncnn.add(tf.keras.layers.Dense(units=1, activation='sigmoid'))\n\n# Part 3 - Training the CNN\n\n# Compiling the CNN\ncnn.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])\nes = tf.keras.callbacks.EarlyStopping(\n    monitor=\"val_loss\",\n    patience=3\n)\n# Training the CNN on the Training set and evaluating it on the Test set\nhistory = cnn.fit(x = training_set, validation_data = test_set, epochs = 25, callbacks=[es])\nhistory.history['val_loss']'''","metadata":{"execution":{"iopub.status.busy":"2021-06-02T07:52:23.911262Z","iopub.execute_input":"2021-06-02T07:52:23.911784Z","iopub.status.idle":"2021-06-02T08:20:05.897658Z","shell.execute_reply.started":"2021-06-02T07:52:23.911751Z","shell.execute_reply":"2021-06-02T08:20:05.896589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n#loss: 0.2263 - accuracy: 0.9010 - val_loss: 0.2897 - val_accuracy: 0.8783\n# Building the CNN\nfrom keras.models import Model,Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, Dropout, Dense,Flatten\n# Initialising the CNN\ncnn = tf.keras.models.Sequential()\n\n# Step 1 - Convolution\ncnn.add(tf.keras.layers.Conv2D(filters=32, kernel_size=3, activation='relu', input_shape=[64, 64, 3]))\ncnn.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))\n# Adding a second convolutional layer\ncnn.add(tf.keras.layers.Conv2D(filters=64, kernel_size=3, activation='relu'))\ncnn.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))\ncnn.add(tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation='relu'))\ncnn.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))\ncnn.add(Dropout(0.2))\ncnn.add(tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation='relu'))\ncnn.add(tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation='relu'))\ncnn.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))\n\n# Step 3 - Flattening\ncnn.add(tf.keras.layers.Flatten())\n\n# Step 4 - Full Connection\ncnn.add(tf.keras.layers.Dense(units=128, activation='relu'))\ncnn.add(tf.keras.layers.Dense(units=128, activation='relu'))\ncnn.add(Dropout(0.2))\n# Step 5 - Output Layer\ncnn.add(tf.keras.layers.Dense(units=1, activation='sigmoid'))\n\n# Part 3 - Training the CNN\n\n# Compiling the CNN\ncnn.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])\nes = tf.keras.callbacks.EarlyStopping(\n    monitor=\"val_loss\",\n    patience=3\n)\n# Training the CNN on the Training set and evaluating it on the Test set\nhistory = cnn.fit(x = training_set, validation_data = test_set, epochs = 25, callbacks=[es])\nhistory.history['val_loss']\n\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''predict_datagen = ImageDataGenerator(rescale = 1./255)\npredict_set = predict_datagen.flow_from_dataframe(df,directory='/kaggle/temp/test1',x_col=\"filename\", y_col=None,\n                                            target_size = (64, 64),\n                                            batch_size = 32,\n                                            class_mode = None,\n                                            shuffle=False)\npred_out=cnn.predict_generator(predict_set,steps = df.shape[0])\ntraining_set.class_indices\nlabels= [1 if output > 0.5 else 0 for output in pred_out]\n#labels\ndf['label']=labels\ndf.drop(columns=['filename'],inplace=True)\ndf.to_csv('submission1.csv', index=False)'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#using Yolo for prediction\nimport os\nimport requests\ndef download(url: str, dest_folder: str):\n    if not os.path.exists(dest_folder):\n        os.makedirs(dest_folder)  # create folder if it does not exist\n\n    filename = url.split('/')[-1].replace(\" \", \"_\")  # be careful with file names\n    file_path = os.path.join(dest_folder, filename)\n\n    r = requests.get(url, stream=True)\n    if r.ok:\n        print(\"saving to\", os.path.abspath(file_path))\n        with open(file_path, 'wb') as f:\n            for chunk in r.iter_content(chunk_size=1024 * 8):\n                if chunk:\n                    f.write(chunk)\n                    f.flush()\n                    os.fsync(f.fileno())\n    else:  # HTTP status code 4XX/5XX\n        print(\"Download failed: status code {}\\n{}\".format(r.status_code, r.text))\n\n\ndownload(\"https://pjreddie.com/media/files/yolov3.weights\", dest_folder=\"/kaggle/temp/weights\")\nfilenames = os.listdir('/kaggle/temp/weights') \nprint(filenames)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:17:32.535016Z","iopub.execute_input":"2021-06-25T15:17:32.535434Z","iopub.status.idle":"2021-06-25T15:19:38.683519Z","shell.execute_reply.started":"2021-06-25T15:17:32.535345Z","shell.execute_reply":"2021-06-25T15:19:38.682379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"download(\"https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov3.cfg\", dest_folder=\"/kaggle/temp/weights\")\nfilenames = os.listdir('/kaggle/temp/weights') \nprint(filenames)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:26:02.979307Z","iopub.execute_input":"2021-06-25T15:26:02.979674Z","iopub.status.idle":"2021-06-25T15:26:03.244043Z","shell.execute_reply.started":"2021-06-25T15:26:02.979642Z","shell.execute_reply":"2021-06-25T15:26:03.242626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zip = zipfile.ZipFile('/kaggle/input/dogs-vs-cats/test1.zip')\nzip.extractall('/kaggle/temp/')\nfilenames = os.listdir('/kaggle/temp/test1') \n''' \n'10435.jpg',\n '6790.jpg',\n '4644.jpg',\n '1835.jpg',\n '1102.jpg'\n '''\n# list available files in the container\n#print (zip.namelist())","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:28:15.960568Z","iopub.execute_input":"2021-06-25T15:28:15.961002Z","iopub.status.idle":"2021-06-25T15:28:23.777636Z","shell.execute_reply.started":"2021-06-25T15:28:15.960945Z","shell.execute_reply":"2021-06-25T15:28:23.776645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''example: '''\nimport re\nid = []\nfor name in filenames:\n    id.append(name.split('.')[0])\n    \ndf = pd.DataFrame.from_dict({'id':id,'filename':filenames})\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:31:58.559055Z","iopub.execute_input":"2021-06-25T15:31:58.559405Z","iopub.status.idle":"2021-06-25T15:31:58.602471Z","shell.execute_reply.started":"2021-06-25T15:31:58.559375Z","shell.execute_reply":"2021-06-25T15:31:58.601435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set of 80 class labels which YOLO is trained\nclass_labels = [\"person\",\"bicycle\",\"car\",\"motorcycle\",\"airplane\",\"bus\",\"train\",\"truck\",\"boat\",\n                \"trafficlight\",\"firehydrant\",\"stopsign\",\"parkingmeter\",\"bench\",\"bird\",\"cat\",\n                \"dog\",\"horse\",\"sheep\",\"cow\",\"elephant\",\"bear\",\"zebra\",\"giraffe\",\"backpack\",\n                \"umbrella\",\"handbag\",\"tie\",\"suitcase\",\"frisbee\",\"skis\",\"snowboard\",\"sportsball\",\n                \"kite\",\"baseballbat\",\"baseballglove\",\"skateboard\",\"surfboard\",\"tennisracket\",\n                \"bottle\",\"wineglass\",\"cup\",\"fork\",\"knife\",\"spoon\",\"bowl\",\"banana\",\"apple\",\n                \"sandwich\",\"orange\",\"broccoli\",\"carrot\",\"hotdog\",\"pizza\",\"donut\",\"cake\",\"chair\",\n                \"sofa\",\"pottedplant\",\"bed\",\"diningtable\",\"toilet\",\"tvmonitor\",\"laptop\",\"mouse\",\n                \"remote\",\"keyboard\",\"cellphone\",\"microwave\",\"oven\",\"toaster\",\"sink\",\"refrigerator\",\n                \"book\",\"clock\",\"vase\",\"scissors\",\"teddybear\",\"hairdrier\",\"toothbrush\"]\n\nyolo_model = cv2.dnn.readNetFromDarknet('/kaggle/temp/weights/yolov3.cfg','/kaggle/temp/weights/yolov3.weights')\nyolo_layers = yolo_model.getLayerNames()\nyolo_output_layer = [yolo_layers[yolo_layer[0] - 1] for yolo_layer in yolo_model.getUnconnectedOutLayers()]","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:42:04.690766Z","iopub.execute_input":"2021-06-25T15:42:04.691132Z","iopub.status.idle":"2021-06-25T15:42:04.946543Z","shell.execute_reply.started":"2021-06-25T15:42:04.691099Z","shell.execute_reply":"2021-06-25T15:42:04.945582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(image):\n    img_to_detect = cv2.imread('/kaggle/temp/test1/'+image)\n    img_height = img_to_detect.shape[0]\n    img_width = img_to_detect.shape[1]\n    # convert to blob to pass into model\n    img_blob = cv2.dnn.blobFromImage(img_to_detect, 0.003922, (416, 416), swapRB=True, crop=False)\n    # input preprocessed blob into model and pass through the model\n    yolo_model.setInput(img_blob)\n    # obtain the detection layers by forwarding through till the output layer\n    obj_detection_layers = yolo_model.forward(yolo_output_layer)\n    label = -1\n    # loop over each of the layer outputs\n    for object_detection_layer in obj_detection_layers:\n        # loop over the detections\n        for object_detection in object_detection_layer:\n\n            # obj_detections[1 to 4] => will have the two center points, box width and box height\n            # obj_detections[5] => will have scores for all objects within bounding box\n            all_scores = object_detection[5:]\n            predicted_class_id = np.argmax(all_scores)\n            prediction_confidence = all_scores[predicted_class_id]\n\n            # take only predictions with confidence more than 20%\n            if prediction_confidence > 0.70:\n                #get the predicted label\n                predicted_class_label = class_labels[predicted_class_id]\n                if(predicted_class_label=='cat'):\n                    label= 0\n                else: \n                    label= 1\n    return label","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:48:28.026296Z","iopub.execute_input":"2021-06-25T15:48:28.026695Z","iopub.status.idle":"2021-06-25T15:48:28.037355Z","shell.execute_reply.started":"2021-06-25T15:48:28.026664Z","shell.execute_reply":"2021-06-25T15:48:28.036574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict('10435.jpg')","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:45:08.455998Z","iopub.execute_input":"2021-06-25T15:45:08.456528Z","iopub.status.idle":"2021-06-25T15:45:09.150134Z","shell.execute_reply.started":"2021-06-25T15:45:08.456493Z","shell.execute_reply":"2021-06-25T15:45:09.149163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['label']=df.filename.apply(lambda x: predict(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-25T15:48:33.224067Z","iopub.execute_input":"2021-06-25T15:48:33.224412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:39:20.138099Z","iopub.execute_input":"2021-06-02T08:39:20.138623Z","iopub.status.idle":"2021-06-02T08:39:20.147115Z","shell.execute_reply.started":"2021-06-02T08:39:20.13859Z","shell.execute_reply":"2021-06-02T08:39:20.146399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.remove('/kaggle/working/*.*') ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.drop(columns=['filename'],inplace=True)\ndf.to_csv('submission2.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:39:25.074711Z","iopub.execute_input":"2021-06-02T08:39:25.075239Z","iopub.status.idle":"2021-06-02T08:39:25.102795Z","shell.execute_reply.started":"2021-06-02T08:39:25.075206Z","shell.execute_reply":"2021-06-02T08:39:25.101974Z"},"trusted":true},"execution_count":null,"outputs":[]}]}