{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Extensive data\n I created a larger monet dataset by uploading an image from wikiart. And then convert it to TFRecords. Hope it helps your work\n\nDataset: https://www.kaggle.com/doanquanvietnamca/monet-ext-data\n![monet](https://camo.githubusercontent.com/1862adecd202ba420847653d9a119f9fed9a3abd/68747470733a2f2f6a756e79616e7a2e6769746875622e696f2f4379636c6547414e2f696d616765732f70686f746f327061696e74696e672e6a7067)","metadata":{}},{"cell_type":"markdown","source":"## Script to download from WikiArt\n```\ndef provide_monet_dataset():\n    # Dowload the script that will manage the data procurement\n    # https://github.com/lucasdavid/wikiart\n    os.system(\"git clone https://github.com/lucasdavid/wikiart.git\")\n    os.system(\"python3 wikiart/wikiart.py --datadir ./wikiart-saved/ fetch --only artists\")\n\n    # Metadata to indicate the artist to be downloaded\n    entry = '''[{\n        \"artistName\": \"Claude Monet\",\n        \"birthDay\": \"/Date(-4074969600000)/\",\n        \"birthDayAsString\": \"November 14, 1840\",\n        \"contentId\": 211667,\n        \"deathDay\": \"/Date(-1359331200000)/\",\n        \"deathDayAsString\": \"December 5, 1926\",\n        \"dictonaries\": [\n            1221,\n            316\n        ],\n        \"image\": \"https://uploads0.wikiart.org/00115/images/claude-monet/440px-claude-monet-1899-nadar-crop.jpg!Portrait.jpg\",\n        \"lastNameFirst\": \"Monet Claude\",\n        \"url\": \"claude-monet\",\n        \"wikipediaUrl\": \"http://en.wikipedia.org/wiki/Claude_Monet\"\n    }]'''\n\n    with open('./wikiart-saved/meta/artists.json', 'w') as artists_file:\n        artists_file.write(entry)\n\n    # Download the art from the artist\n    os.system(\"python3 wikiart/wikiart.py --datadir ./wikiart-saved/ fetch\")\n\n    # Filepath of all files retrieval\n    files = []\n    # r=root, d=directories, f = files\n    for r, d, f in os.walk('wikiart-saved/images/claude-monet'):\n        for file in f:\n            if '.jpg' in file:\n                files.append(os.path.join(r, file))\n\n    images = []\n\n    for file in files:\n        images.append(_preprocess(cv2.imread(file)))\n\n    # An array with al the read fieles is returned\n    return images\n```","metadata":{}},{"cell_type":"code","source":"# LOAD LIBRARIES\nimport numpy as np, pandas as pd, os\nimport matplotlib.pyplot as plt, cv2\nimport tensorflow as tf, re, math","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-01T17:53:13.859121Z","iopub.execute_input":"2022-04-01T17:53:13.859668Z","iopub.status.idle":"2022-04-01T17:53:16.253757Z","shell.execute_reply.started":"2022-04-01T17:53:13.859611Z","shell.execute_reply":"2022-04-01T17:53:16.252488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/oregon-wildlife/oregon_wildlife/oregon_wildlife/","metadata":{"execution":{"iopub.status.busy":"2022-04-01T17:55:34.571854Z","iopub.execute_input":"2022-04-01T17:55:34.572378Z","iopub.status.idle":"2022-04-01T17:55:35.414165Z","shell.execute_reply.started":"2022-04-01T17:55:34.572322Z","shell.execute_reply":"2022-04-01T17:55:35.412815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PATHS TO IMAGES\nPATH = '../input/oregon-wildlife/oregon_wildlife/oregon_wildlife/bald_eagle'\nIMGS = os.listdir(PATH)\nprint('There are %i monet images '%(len(IMGS)))","metadata":{"execution":{"iopub.status.busy":"2022-04-01T18:03:50.444752Z","iopub.execute_input":"2022-04-01T18:03:50.445220Z","iopub.status.idle":"2022-04-01T18:03:50.455875Z","shell.execute_reply.started":"2022-04-01T18:03:50.445171Z","shell.execute_reply":"2022-04-01T18:03:50.454888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-04-01T18:03:51.276795Z","iopub.execute_input":"2022-04-01T18:03:51.277207Z","iopub.status.idle":"2022-04-01T18:03:51.285369Z","shell.execute_reply.started":"2022-04-01T18:03:51.277172Z","shell.execute_reply":"2022-04-01T18:03:51.284106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_example(feature0, feature1, feature2):\n  feature = {\n      'image_name': _bytes_feature(feature0),\n      'image': _bytes_feature(feature1),\n      'target': _bytes_feature(feature2)\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2022-04-01T18:03:51.943837Z","iopub.execute_input":"2022-04-01T18:03:51.944260Z","iopub.status.idle":"2022-04-01T18:03:51.950361Z","shell.execute_reply.started":"2022-04-01T18:03:51.944217Z","shell.execute_reply":"2022-04-01T18:03:51.949116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Write TFRecords","metadata":{}},{"cell_type":"code","source":"\nPATH = '../input/oregon-wildlife/oregon_wildlife/oregon_wildlife/'\n\nfor dirr in os.listdir(PATH):\n    log = 0\n    IMGS = os.listdir(PATH+ dirr)\n    with tf.io.TFRecordWriter(f'wild_{dirr}.tfrec') as writer:\n        for k in range(len(IMGS)):\n            try:\n                img = cv2.imread(f\"../input/oregon-wildlife/oregon_wildlife/oregon_wildlife/{dirr}/\"+IMGS[k])\n\n                h, w, *_ = img.shape \n                if h*w > 1024*1024:\n                    img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 94))[1].tostring()\n                    name = IMGS[k].split('.')[0]\n                    example = serialize_example(\n                        str.encode(name),\n                        img, \n                        b'monet')\n                    writer.write(example)\n                    log+=1\n                if k%100==0: print(log,', ', end='')\n            except:\n                pass\n    print(\"\\n\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}