{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# check what's in the current directory\n!ls","metadata":{"execution":{"iopub.status.busy":"2021-12-01T00:47:47.829256Z","iopub.execute_input":"2021-12-01T00:47:47.829660Z","iopub.status.idle":"2021-12-01T00:47:48.628415Z","shell.execute_reply.started":"2021-12-01T00:47:47.829567Z","shell.execute_reply":"2021-12-01T00:47:48.627013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set up pre-trained weights\nWith internet enabled, PyTorch will check the cache for pre-trained weights as needed and, if the required file is not found, automatically download it. To run without internet, the pre-trained checkpoint file must exist locally, so I've uploaded a pre-trained ResNet-50 backbone as part of my private dataset. We can copy the file to the default torch home location `~/.cache/torch`, and PyTorch will find it there instead of needing to download it.","metadata":{}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nimport shutil\n\nreadonly_checkpoint_dir = Path('/kaggle/input/cmdwheatdet/torch_cache/hub/checkpoints')\nreadwrite_checkpoint_dir = Path('/root/.cache/torch/hub/checkpoints')\nif not (readwrite_checkpoint_dir).is_dir():\n    os.makedirs(readwrite_checkpoint_dir)\ncheckpoint_path = sorted(readonly_checkpoint_dir.glob('*.pth'))[0]\nshutil.copy(checkpoint_path, readwrite_checkpoint_dir)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T00:47:48.630801Z","iopub.execute_input":"2021-12-01T00:47:48.631140Z","iopub.status.idle":"2021-12-01T00:47:51.530702Z","shell.execute_reply.started":"2021-12-01T00:47:48.631099Z","shell.execute_reply":"2021-12-01T00:47:51.529611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Install dependencies\nI'm installing my little Python package that I put together for this Kaggle wheat detection challenge. The source code is here: https://github.com/sheromon/wheat-detection.","metadata":{}},{"cell_type":"code","source":"# I appreciate that when you add a Kaggle dataset, it unzips and untars everything, but it's actually inconvenient\n# for me in this situation where `pip download` give you a tar.gz file and not a wheel.\n# seems like `pip install --find-links <directory-path>` doesn't work when the install file, normally .tar.gz, is unzipped.\n# anyway, this feels silly, but we can fix it by tarring up the untarred files for configobj.\n!mkdir /kaggle/working/deps/\n!cd /kaggle/input/cmdwheatdet/deps/deps/configobj-5.0.6/ && tar -czf /kaggle/working/deps/configobj-5.0.6.tar.gz configobj-5.0.6/","metadata":{"execution":{"iopub.status.busy":"2021-12-01T00:47:55.893213Z","iopub.execute_input":"2021-12-01T00:47:55.894000Z","iopub.status.idle":"2021-12-01T00:47:57.435302Z","shell.execute_reply.started":"2021-12-01T00:47:55.893933Z","shell.execute_reply":"2021-12-01T00:47:57.433787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links /kaggle/input/cmdwheatdet/deps/deps --find-links /kaggle/working/deps/ cmd-wheat-det","metadata":{"execution":{"iopub.status.busy":"2021-12-01T00:48:01.257805Z","iopub.execute_input":"2021-12-01T00:48:01.258749Z","iopub.status.idle":"2021-12-01T00:48:17.836249Z","shell.execute_reply.started":"2021-12-01T00:48:01.258694Z","shell.execute_reply":"2021-12-01T00:48:17.835011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pytorch_lightning as pl\npl.__version__","metadata":{"execution":{"iopub.status.busy":"2021-12-01T00:48:21.738607Z","iopub.execute_input":"2021-12-01T00:48:21.738988Z","iopub.status.idle":"2021-12-01T00:48:25.137075Z","shell.execute_reply.started":"2021-12-01T00:48:21.738945Z","shell.execute_reply":"2021-12-01T00:48:25.135711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nimport pprint\n\nimport numpy as np\nimport torch\n\nfrom wheat.config import load_config\nfrom wheat.scripts import train, evaluate, predict","metadata":{"execution":{"iopub.status.busy":"2021-12-01T00:48:27.165896Z","iopub.execute_input":"2021-12-01T00:48:27.166246Z","iopub.status.idle":"2021-12-01T00:48:30.312848Z","shell.execute_reply.started":"2021-12-01T00:48:27.166213Z","shell.execute_reply":"2021-12-01T00:48:30.311294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the default configuration file\nconfig = load_config()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T00:48:54.798016Z","iopub.execute_input":"2021-12-01T00:48:54.798386Z","iopub.status.idle":"2021-12-01T00:48:54.806425Z","shell.execute_reply.started":"2021-12-01T00:48:54.798345Z","shell.execute_reply":"2021-12-01T00:48:54.805204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config['numpy_seed'] = 1234\nconfig['data_dir'] = '/kaggle/input/global-wheat-detection'","metadata":{"execution":{"iopub.status.busy":"2021-12-01T00:49:09.554920Z","iopub.execute_input":"2021-12-01T00:49:09.555305Z","iopub.status.idle":"2021-12-01T00:49:09.560044Z","shell.execute_reply.started":"2021-12-01T00:49:09.555262Z","shell.execute_reply":"2021-12-01T00:49:09.559099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pp = pprint.PrettyPrinter(indent=2)\npp.pprint(config)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T00:49:11.193546Z","iopub.execute_input":"2021-12-01T00:49:11.194126Z","iopub.status.idle":"2021-12-01T00:49:11.203988Z","shell.execute_reply.started":"2021-12-01T00:49:11.194083Z","shell.execute_reply":"2021-12-01T00:49:11.203153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2021-11-28T21:03:53.187987Z","iopub.execute_input":"2021-11-28T21:03:53.188319Z","iopub.status.idle":"2021-11-28T21:03:53.948545Z","shell.execute_reply.started":"2021-11-28T21:03:53.188286Z","shell.execute_reply":"2021-11-28T21:03:53.947442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set pytorch lightning flags here\npl_args_dict = dict(\n    max_epochs=30,\n    gpus=1,\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T21:20:43.691946Z","iopub.execute_input":"2021-11-28T21:20:43.692415Z","iopub.status.idle":"2021-11-28T21:20:43.695539Z","shell.execute_reply.started":"2021-11-28T21:20:43.692367Z","shell.execute_reply":"2021-11-28T21:20:43.694942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run training!\ntrain.train(config, pl_args_dict)","metadata":{"execution":{"iopub.status.busy":"2021-11-28T21:20:47.868660Z","iopub.execute_input":"2021-11-28T21:20:47.869139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find the last model checkpoint\ncheckpoint_dir = Path('lightning_logs/version_0/checkpoints')\ncheckpoint_files = checkpoint_dir.glob('*.ckpt')\n# there should only be one checkpoint file saved, so just get the first match\ncheckpoint_path = next(checkpoint_files)\ncheckpoint_path","metadata":{"execution":{"iopub.status.busy":"2021-11-21T20:01:29.236459Z","iopub.execute_input":"2021-11-21T20:01:29.23706Z","iopub.status.idle":"2021-11-21T20:01:29.244773Z","shell.execute_reply.started":"2021-11-21T20:01:29.237021Z","shell.execute_reply":"2021-11-21T20:01:29.243895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run evaluation\n# if this environment variable is set, detections and ground truth annotations\n# will be saved to .csv files for easy loading and analysis later on\nos.environ['CMD_WHEAT_OUTPUT_DIR'] = str(checkpoint_dir.parent)\nevaluate.evaluate(config, {'gpus': 1}, checkpoint_path)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run inference\npredict.predict(config, pl_args_dict, checkpoint_path)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T20:01:52.71975Z","iopub.execute_input":"2021-11-21T20:01:52.720482Z","iopub.status.idle":"2021-11-21T20:01:54.990819Z","shell.execute_reply.started":"2021-11-21T20:01:52.720445Z","shell.execute_reply":"2021-11-21T20:01:54.989844Z"},"trusted":true},"execution_count":null,"outputs":[]}]}