{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Machine Learning Security Capture The Flag Competition\n\n![ss](https://miro.medium.com/max/1400/1*iMvquhEkvmjhEoiNfKOqXg.png)\n\nWelcome to my attempt at solving the DEFCON AI Village Capture-the-Flag (CTF) challenges.\n\n\n## Intro\n\nIn this notebook, I'll show and explain the solutions I came up with.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport requests\nfrom IPython.display import Image as Ima\nfrom IPython.display import clear_output, display\nfrom PIL import Image as Im\nfrom torch.utils.data import Dataset, DataLoader\nimport torch\nimport joblib\nimport matplotlib.pyplot as plt\nimport json\nimport base64\nimport shutil\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndir_name = \"/kaggle/input/ai-village-ctf\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-01T20:21:41.085194Z","iopub.execute_input":"2022-09-01T20:21:41.086258Z","iopub.status.idle":"2022-09-01T20:21:42.885514Z","shell.execute_reply.started":"2022-09-01T20:21:41.086132Z","shell.execute_reply":"2022-09-01T20:21:42.884531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.image import imread\n\n# Hotdog\nfrom io import BytesIO\n\n# Math\nfrom sklearn.cluster import MeanShift\nfrom sklearn.decomposition import PCA\nfrom collections import Counter\n\n# Wifi\nfrom sklearn.manifold import LocallyLinearEmbedding\nfrom sklearn.neighbors import NearestNeighbors\nfrom sklearn.preprocessing import MinMaxScaler\nfrom matplotlib import offsetbox\n\n# Inference\nimport tqdm\nimport cv2\n\nfrom collections import namedtuple\n\nimport torch\nimport torchvision.transforms as transforms\nfrom torchvision.datasets import EMNIST, MNIST\nfrom tqdm.notebook import tqdm, trange\n\n! pip install torchsummary \nfrom torchsummary import summary\n\n# Forensics\nfrom tensorflow import keras\n\n# Deepfake\nfrom IPython.display import HTML\nfrom base64 import b64encode\n\n# Murderbots\nimport seaborn as sns\n\nfrom sklearn.feature_selection import mutual_info_classif\nfrom sklearn.dummy import DummyClassifier\n\nfrom sklearn.feature_selection import SelectFromModel, SelectKBest\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC, NuSVC\n\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import roc_curve, roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:21:42.889076Z","iopub.execute_input":"2022-09-01T20:21:42.889759Z","iopub.status.idle":"2022-09-01T20:21:59.993272Z","shell.execute_reply.started":"2022-09-01T20:21:42.889729Z","shell.execute_reply":"2022-09-01T20:21:59.992268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# A helper function for tracking flags. You could do it yourself, but why?\n\ndef add_flag(challenge_id: str, flag: str) -> str:\n    try:\n        submission_path = \"/kaggle/working/submission.csv\"\n        template_path = f\"{dir_name}/submission.csv\"\n        if not os.path.isfile(submission_path) :\n            shutil.copyfile(template_path, submission_path)\n        flags = pd.read_csv(submission_path, index_col=\"challenge_id\")\n        valid = flags.index.to_list()\n        if challenge_id in valid:\n            flags.at[challenge_id] = flag\n            flags.to_csv(submission_path)\n            print(\"Flag saved. If you want to update the scoreboard, click 'submit' on the right. Remember that you can only update your score on the scoreboard 50 times per day\")\n        else:\n            print(\"Invalid challenge_id\")\n    except:\n        print(\"We had trouble saving that flag. Keep a copy of it and contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:21:59.994741Z","iopub.execute_input":"2022-09-01T20:21:59.995093Z","iopub.status.idle":"2022-09-01T20:22:00.003460Z","shell.execute_reply.started":"2022-09-01T20:21:59.995059Z","shell.execute_reply":"2022-09-01T20:22:00.002413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# HOTDOG (id: hotdog) -- 200 Points\n\n### Scenario\n\nThis is Chester. He's heard of Hotdog/Not Hotdog, but really wants to be classified as a hot-dog (get it?).\n\n### Objective\n\nHelp Chester classify as a hotdog.\n\n## Method\n\nHaving no constraint on the type of images we can give the API, this challenge is straightforward. We will give the model a generic picture of a hotdog.","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2)\n\nIMAGE_DIMS = (224, 224)\n\n# Original Image\nimg = Im.open(f\"{dir_name}/hotdog/chester.png\")\nimg = img.resize(IMAGE_DIMS)\nimg_bytes = img.tobytes()\nb64_img_bytes = base64.urlsafe_b64encode(img_bytes)\n\ntry:\n    r = requests.post(\"https://hotdog.fly.dev/score\", data=json.dumps({\"input\": b64_img_bytes.decode()}))\n    print(r.text)\n    axs[0].imshow(img)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")\n\n# Generic hotdog Image\nresponse = requests.get('https://i0.wp.com/www.foodlovinfamily.com/wp-content/uploads/2020/07/how-to-boil-a-hotdog-square.jpg')\nimg = Im.open(BytesIO(response.content))\nimg = img.resize(IMAGE_DIMS)\nimg_bytes = img.tobytes()\nb64_img_bytes = base64.urlsafe_b64encode(img_bytes)\n\ntry:\n    r = requests.post(\"https://hotdog.fly.dev/score\", data=json.dumps({\"input\": b64_img_bytes.decode()}))\n    print(r.text)\n    axs[1].imshow(img)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:00.006486Z","iopub.execute_input":"2022-09-01T20:22:00.007232Z","iopub.status.idle":"2022-09-01T20:22:01.175804Z","shell.execute_reply.started":"2022-09-01T20:22:00.007196Z","shell.execute_reply":"2022-09-01T20:22:01.174848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"hotdog\", json.loads(r.text)['message'])","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:01.176806Z","iopub.execute_input":"2022-09-01T20:22:01.177178Z","iopub.status.idle":"2022-09-01T20:22:01.204058Z","shell.execute_reply.started":"2022-09-01T20:22:01.177142Z","shell.execute_reply":"2022-09-01T20:22:01.203058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Math\n\nThis series of \"Math\" challenges is designed to introduce the concepts of dimensionality.\n\nData is often represented in a high dimensional space, but is actually several intrinsically low dimensional components that can be diagnosed. Intrinsic low-dimensionailty is incredibly important to machine learning because many ML algorithms rely on it. Medical EKG data is as single intrinsically 2 dimensional component, despite often being represented as 100 dimensions or more. The classic MNIST dataset is several components of 8-10 dimensions, depending on the digits class, despite being represented in 784 dimensions. The 4 math challenges are focused on this problem.","metadata":{}},{"cell_type":"markdown","source":"# Math Challenge 1 (id: math_1) -- 100 Points\n\nHow many clusters are in the clusters1.npy, clusters2.npy, and clusters3.npy? The key is the number of clusters in order, with no spaces. These files are available in the input directory under `math`.\n\n## Method\n\nClustering is the process of categorizing samples into similar groups. The categories are determined by the distribution of the data. In this challenge we have no labels so we will resort to unsupervised techniques. We will compute the number of clusters with [Mean Shift](https://scikit-learn.org/stable/modules/generated/sklearn.cluster.MeanShift.html) on each dataset. Since we don't know the number of clusters ahead of time, we need an algorithm that does not need it as parameter. That is one advantage of Meanshift. I find the correct number of clusters by playing with the parameter (the bandwidth) and finding a stable number of clusters.","metadata":{}},{"cell_type":"code","source":"np.load(f'{dir_name}/math/clusters1.npy').shape, np.load(f'{dir_name}/math/clusters2.npy').shape, np.load(f'{dir_name}/math/clusters3.npy').shape","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:01.205706Z","iopub.execute_input":"2022-09-01T20:22:01.206065Z","iopub.status.idle":"2022-09-01T20:22:01.234639Z","shell.execute_reply.started":"2022-09-01T20:22:01.206032Z","shell.execute_reply":"2022-09-01T20:22:01.233625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clusters1 = np.load(f'{dir_name}/math/clusters1.npy')\n\nms = MeanShift(bandwidth=9, bin_seeding=False)\nms.fit(clusters1)\n\nlabels = ms.labels_\n\nlabels_unique = np.unique(labels)\nn_clusters_ = len(labels_unique)\n\nprint(\"number of estimated clusters : %d\" % n_clusters_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:01.236340Z","iopub.execute_input":"2022-09-01T20:22:01.236671Z","iopub.status.idle":"2022-09-01T20:22:01.357804Z","shell.execute_reply.started":"2022-09-01T20:22:01.236639Z","shell.execute_reply":"2022-09-01T20:22:01.356779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clusters2 = np.load(f'{dir_name}/math/clusters2.npy')\n\nms = MeanShift(bandwidth=9, bin_seeding=False)\nms.fit(clusters2)\n\nlabels = ms.labels_\n\nlabels_unique = np.unique(labels)\nn_clusters_ = len(labels_unique)\n\nprint(\"number of estimated clusters : %d\" % n_clusters_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:01.359203Z","iopub.execute_input":"2022-09-01T20:22:01.359618Z","iopub.status.idle":"2022-09-01T20:22:01.434605Z","shell.execute_reply.started":"2022-09-01T20:22:01.359583Z","shell.execute_reply":"2022-09-01T20:22:01.433756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clusters3 = np.load(f'{dir_name}/math/clusters3.npy')\n\nms = MeanShift(bandwidth=8, bin_seeding=False)\nms.fit(clusters3)\n\nlabels = ms.labels_\n\nlabels_unique = np.unique(labels)\nn_clusters_ = len(labels_unique)\n\nprint(\"number of estimated clusters : %d\" % n_clusters_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:01.435913Z","iopub.execute_input":"2022-09-01T20:22:01.437338Z","iopub.status.idle":"2022-09-01T20:22:01.490828Z","shell.execute_reply.started":"2022-09-01T20:22:01.437303Z","shell.execute_reply":"2022-09-01T20:22:01.489874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"math_1\", \"submission\": \"523\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:01.494705Z","iopub.execute_input":"2022-09-01T20:22:01.494963Z","iopub.status.idle":"2022-09-01T20:22:01.624252Z","shell.execute_reply.started":"2022-09-01T20:22:01.494940Z","shell.execute_reply":"2022-09-01T20:22:01.623247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"math_1\", '4IHMQYZWOH4XRV5QS83QY0LB9J5{TB5NUBJFLN4P0UWJH7ELCSSLGTOKQTAG4VT3}TLZQ7JWBE6C5H3CS1CPPJFGFYQLV42P8BA2T7B166WS8MCVPCN72YIFQN7N5C5N')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:01.625816Z","iopub.execute_input":"2022-09-01T20:22:01.626166Z","iopub.status.idle":"2022-09-01T20:22:01.636298Z","shell.execute_reply.started":"2022-09-01T20:22:01.626133Z","shell.execute_reply":"2022-09-01T20:22:01.635303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Math Challenge 2 (id: math_2) -- 200 Points\n\nWhat's the dimensionality of the data in first_dim1.npy, first_dim2.npy, and first_dim3.npy? The key is the number of dimensions in order, with no spaces.  These files are available in the input directory under `math`.\n\n### Method\n\nWhile the *dimension* of data refers to the number of features it has, *dimensionality* refers to the number of essential dimensions it has. To solve this challenge, I reduce the dimensions of each dataset and find how many features are needed to represent the datasets. I used Principal Component Analysis which finds a new coordinate space for the data. The dimensionality of the data is the minimum number of principal components (or features in the new coordinate space) that still retain the maximum information. The explained variance ratio estimates the retained information well and luckily there is a considerable dropoff for each dataset.","metadata":{}},{"cell_type":"code","source":"pca = PCA(n_components=10)\npca.fit(np.load(f'{dir_name}/math/first_dim1.npy'))\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:01.637616Z","iopub.execute_input":"2022-09-01T20:22:01.638505Z","iopub.status.idle":"2022-09-01T20:22:01.912192Z","shell.execute_reply.started":"2022-09-01T20:22:01.638468Z","shell.execute_reply":"2022-09-01T20:22:01.911159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=9)\npca.fit(np.load(f'{dir_name}/math/first_dim2.npy'))\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:01.913588Z","iopub.execute_input":"2022-09-01T20:22:01.913915Z","iopub.status.idle":"2022-09-01T20:22:02.123048Z","shell.execute_reply.started":"2022-09-01T20:22:01.913883Z","shell.execute_reply":"2022-09-01T20:22:02.122025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=8)\npca.fit(np.load(f'{dir_name}/math/first_dim3.npy'))\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:02.124589Z","iopub.execute_input":"2022-09-01T20:22:02.124911Z","iopub.status.idle":"2022-09-01T20:22:02.325266Z","shell.execute_reply.started":"2022-09-01T20:22:02.124878Z","shell.execute_reply":"2022-09-01T20:22:02.324421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"math_2\", \"submission\": \"354\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:02.327074Z","iopub.execute_input":"2022-09-01T20:22:02.327656Z","iopub.status.idle":"2022-09-01T20:22:02.465558Z","shell.execute_reply.started":"2022-09-01T20:22:02.327621Z","shell.execute_reply":"2022-09-01T20:22:02.464588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"math_2\", 'KDSYZMGZGTLW16TNOFSO9DY29L}EMI03JTF640N1PQT2YB6Y1SS7VKSTARCM60I1AMMZERS3IUFC3PO0SWESVVSELLXH12TJADZTWN6M5S5J0PGY8{KEF9JWAYF6QCY2')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:02.466948Z","iopub.execute_input":"2022-09-01T20:22:02.467380Z","iopub.status.idle":"2022-09-01T20:22:02.478282Z","shell.execute_reply.started":"2022-09-01T20:22:02.467345Z","shell.execute_reply":"2022-09-01T20:22:02.477376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Math Challenge 3 (id: math_3) -- 300 Points\n\nWhat's the dimensionality of the data in second_dim1.npy, second_dim2.npy, and second_dim3.npy? The key is the number of clusters in order, with no spaces.  These files are available in the input directory under `math`.\n\n## Method\n\nFollow the same method as for challenge **math_2**.","metadata":{}},{"cell_type":"code","source":"np.load(f'{dir_name}/math/second_dim1.npy').shape, np.load(f'{dir_name}/math/second_dim2.npy').shape, np.load(f'{dir_name}/math/second_dim3.npy').shape","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:02.479487Z","iopub.execute_input":"2022-09-01T20:22:02.479738Z","iopub.status.idle":"2022-09-01T20:22:02.507726Z","shell.execute_reply.started":"2022-09-01T20:22:02.479715Z","shell.execute_reply":"2022-09-01T20:22:02.506918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"second_dim1 = np.load(f'{dir_name}/math/second_dim1.npy')\n\npca = PCA(n_components=10)\npca.fit(second_dim1)\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:02.509949Z","iopub.execute_input":"2022-09-01T20:22:02.510547Z","iopub.status.idle":"2022-09-01T20:22:02.688187Z","shell.execute_reply.started":"2022-09-01T20:22:02.510513Z","shell.execute_reply":"2022-09-01T20:22:02.687171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"second_dim2 = np.load(f'{dir_name}/math/second_dim2.npy')\n\npca = PCA(n_components=9)\npca.fit(second_dim2)\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:02.689332Z","iopub.execute_input":"2022-09-01T20:22:02.689643Z","iopub.status.idle":"2022-09-01T20:22:02.880470Z","shell.execute_reply.started":"2022-09-01T20:22:02.689611Z","shell.execute_reply":"2022-09-01T20:22:02.879519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"second_dim3 = np.load(f'{dir_name}/math/second_dim3.npy')\n\npca = PCA(n_components=8)\npca.fit(second_dim3)\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:02.881762Z","iopub.execute_input":"2022-09-01T20:22:02.883358Z","iopub.status.idle":"2022-09-01T20:22:03.075046Z","shell.execute_reply.started":"2022-09-01T20:22:02.883320Z","shell.execute_reply":"2022-09-01T20:22:03.074097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"math_3\", \"submission\": \"474\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:03.076549Z","iopub.execute_input":"2022-09-01T20:22:03.076875Z","iopub.status.idle":"2022-09-01T20:22:03.213026Z","shell.execute_reply.started":"2022-09-01T20:22:03.076842Z","shell.execute_reply":"2022-09-01T20:22:03.212001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"math_3\", 'GDNVL8FZYH1WR0JI0XQ4OLKOH}{FK9VFCAMSE7RB4TDM0UMCAJTM6GM6ZYVL066UTKROIXNV4NV_AXAK3L0JAB2I21WYHGPAB6NVZTX07V2BQGY1PPQ22TXKAZTMWK85')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:03.214380Z","iopub.execute_input":"2022-09-01T20:22:03.214803Z","iopub.status.idle":"2022-09-01T20:22:03.225425Z","shell.execute_reply.started":"2022-09-01T20:22:03.214756Z","shell.execute_reply":"2022-09-01T20:22:03.224380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Math Challenge 4 (id: math_4) -- 400 Points\n\nWhat's the dimensionality of the clusters in clusters1.npy? The key is the dimensions ordered by cluster size (smallest to largest), with no spaces.  These files are available in the input directory under `math`.\n\n## Method\n\nThis challenge is about combining the steps of previous challenges. I cluster the datasets using the method described before (using Mean Shift) and then isolate clusters to compute their individual dimensionality (using PCA).","metadata":{}},{"cell_type":"code","source":"clusters1 = np.load(f'{dir_name}/math/clusters1.npy')\n\nms = MeanShift(bandwidth=9, bin_seeding=False)\nms.fit(clusters1)\n\nlabels = ms.labels_\n\nlabels_unique = np.unique(labels)\nn_clusters_ = len(labels_unique)\n\nprint(\"number of estimated clusters : %d\" % n_clusters_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:03.228063Z","iopub.execute_input":"2022-09-01T20:22:03.228396Z","iopub.status.idle":"2022-09-01T20:22:03.351853Z","shell.execute_reply.started":"2022-09-01T20:22:03.228371Z","shell.execute_reply":"2022-09-01T20:22:03.350834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Counter(ms.labels_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:03.353100Z","iopub.execute_input":"2022-09-01T20:22:03.354132Z","iopub.status.idle":"2022-09-01T20:22:03.361224Z","shell.execute_reply.started":"2022-09-01T20:22:03.354079Z","shell.execute_reply":"2022-09-01T20:22:03.360176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cluster1 = clusters1[labels == 4]\n\npca = PCA(n_components=10)\npca.fit(cluster1)\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:03.362598Z","iopub.execute_input":"2022-09-01T20:22:03.363106Z","iopub.status.idle":"2022-09-01T20:22:03.550088Z","shell.execute_reply.started":"2022-09-01T20:22:03.363066Z","shell.execute_reply":"2022-09-01T20:22:03.549083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cluster2 = clusters1[labels == 3]\n\npca = PCA(n_components=10)\npca.fit(cluster2)\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:03.551681Z","iopub.execute_input":"2022-09-01T20:22:03.552020Z","iopub.status.idle":"2022-09-01T20:22:03.738192Z","shell.execute_reply.started":"2022-09-01T20:22:03.551986Z","shell.execute_reply":"2022-09-01T20:22:03.737312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cluster3 = clusters1[labels == 2]\n\npca = PCA(n_components=10)\npca.fit(cluster3)\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:03.739475Z","iopub.execute_input":"2022-09-01T20:22:03.739799Z","iopub.status.idle":"2022-09-01T20:22:03.916207Z","shell.execute_reply.started":"2022-09-01T20:22:03.739767Z","shell.execute_reply":"2022-09-01T20:22:03.915365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cluster4 = clusters1[labels == 1]\n\npca = PCA(n_components=10)\npca.fit(cluster4)\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:03.924592Z","iopub.execute_input":"2022-09-01T20:22:03.924915Z","iopub.status.idle":"2022-09-01T20:22:04.112180Z","shell.execute_reply.started":"2022-09-01T20:22:03.924890Z","shell.execute_reply":"2022-09-01T20:22:04.111162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cluster5 = clusters1[labels == 0]\n\npca = PCA(n_components=10)\npca.fit(cluster5)\n\nprint(pca.explained_variance_ratio_)\nplt.plot(np.arange(pca.explained_variance_ratio_.shape[0]), pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:04.113428Z","iopub.execute_input":"2022-09-01T20:22:04.113760Z","iopub.status.idle":"2022-09-01T20:22:04.466003Z","shell.execute_reply.started":"2022-09-01T20:22:04.113727Z","shell.execute_reply":"2022-09-01T20:22:04.465035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"math_4\", \"submission\": \"54312\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:04.467679Z","iopub.execute_input":"2022-09-01T20:22:04.468033Z","iopub.status.idle":"2022-09-01T20:22:04.604020Z","shell.execute_reply.started":"2022-09-01T20:22:04.467998Z","shell.execute_reply":"2022-09-01T20:22:04.603031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"math_4\", 'GCJJ52GXGGCFSE6FRZ4K9FYXNJHJLHWTUN4}48HW7HBDB99LP1JT9RIFRPU86426O6I4FVRVD4V12EDGNA91E99C46UQORTOOM1{MYJIXXSRR3TMMI6ZZL2AAGSHIMZG')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:04.605681Z","iopub.execute_input":"2022-09-01T20:22:04.606024Z","iopub.status.idle":"2022-09-01T20:22:04.616430Z","shell.execute_reply.started":"2022-09-01T20:22:04.605990Z","shell.execute_reply":"2022-09-01T20:22:04.615388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WIFI (id: wifi) -- 200 Points\n\n### Scenario\n\nYou really need to check your email, unfortunately you don't know the password.  Fortunately, someone wrote it down.  Unfortunately, it's written down on a low-dimensional manifold embedded in a very high-dimensional space.  Check out the `wifi/Embedded characters.npz` file -- a list of tokens is given in the `tokens` key with their corresponding embeddings in the same order under the `embeddings` key -- and recover the password.\n\n### Objective\n\nThe password is your flag, like all others, it has the format `FLAG{something}`\n\n## Method\n\nThe data is said to be written down in a very high-dimensional space so I will represent the embeddings in lower dimension. Once the data is in lower dimension, I will retrieve the password.","metadata":{}},{"cell_type":"code","source":"wifi = np.load(f'{dir_name}/wifi/Embedded_characters.npz')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:04.618757Z","iopub.execute_input":"2022-09-01T20:22:04.619301Z","iopub.status.idle":"2022-09-01T20:22:04.653883Z","shell.execute_reply.started":"2022-09-01T20:22:04.619265Z","shell.execute_reply":"2022-09-01T20:22:04.652984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SOURCE: https://scikit-learn.org/stable/modules/manifold.html\n\nn_neighbors = 30\n\nembeddings = {\n    \"Standard LLE embedding\": LocallyLinearEmbedding(\n        n_neighbors=n_neighbors, n_components=2, method=\"standard\"\n    ),\n}","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:04.655374Z","iopub.execute_input":"2022-09-01T20:22:04.655792Z","iopub.status.idle":"2022-09-01T20:22:04.661328Z","shell.execute_reply.started":"2022-09-01T20:22:04.655748Z","shell.execute_reply":"2022-09-01T20:22:04.660062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"projections, timing = {}, {}\nfor name, transformer in embeddings.items():\n    if name.startswith(\"Linear Discriminant Analysis\"):\n        data = wifi['embeddings'].copy()\n        data.flat[:: wifi['embeddings'].shape[1] + 1] += 0.01  # Make X invertible\n    else:\n        data = wifi['embeddings']\n\n    print(f\"Computing {name}...\")\n    projections[name] = transformer.fit_transform(data, _)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:04.663099Z","iopub.execute_input":"2022-09-01T20:22:04.663827Z","iopub.status.idle":"2022-09-01T20:22:04.772385Z","shell.execute_reply.started":"2022-09-01T20:22:04.663792Z","shell.execute_reply":"2022-09-01T20:22:04.769754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No patterns is visible at this point but we can try to visualize the data and look for clues.","metadata":{}},{"cell_type":"code","source":"tokens = '!!\"\"##$$%%&&\\'\\'(())**++,,--..//00112233445566778899::;;<<==>>??@@AABBCCDDEEFFGGHHIIJJKKLLMMNNOOPPQQRRSSTTUUVVWWXXYYZZ[[\\\\\\\\]]^^__``aabbccddeeffgghhiijjkkllmmnnooppqqrrssttuuvvwwxxyyzz{{||}}~~'\n\ndef plot_embedding(X, title):\n    fig, ax = plt.subplots()\n    X = MinMaxScaler().fit_transform(X)\n\n    for i, token in enumerate(tokens):\n        ax.scatter(\n            *X[i].T,\n            marker=f\"\"\"${i}$\"\"\",\n            s=60,\n            alpha=0.425,\n            zorder=2,\n        )\n\n    ax.set_title(title)\n    ax.axis(\"off\")\n","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:29:17.307074Z","iopub.execute_input":"2022-09-01T20:29:17.307799Z","iopub.status.idle":"2022-09-01T20:29:17.313904Z","shell.execute_reply.started":"2022-09-01T20:29:17.307755Z","shell.execute_reply":"2022-09-01T20:29:17.312827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name in projections:\n    title = f\"{name}\"\n    plot_embedding(projections[name], title)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:31:28.045333Z","iopub.execute_input":"2022-09-01T20:31:28.045684Z","iopub.status.idle":"2022-09-01T20:31:32.820505Z","shell.execute_reply.started":"2022-09-01T20:31:28.045655Z","shell.execute_reply":"2022-09-01T20:31:32.819292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is an obvious spiraling pattern. Knowing that the flag we are looking for is in the form *FLAG{XXX}*, we want to find the coordinates of letter *F*. Then we can look at neighbooring letters and hope they will spell a flag. There are two tokens for the letter *F*, and trial and error finds that the one indexed at 75 is in the middle of the spiral !","metadata":{}},{"cell_type":"code","source":"def backprop(embedings):\n    flag = \"F\"\n    nearest_n = []\n    next_letter = 75\n    nearest_n.append(next_letter)\n    \n    neigh = NearestNeighbors(n_neighbors=2, radius=0.4)\n    neigh.fit(embedings)\n    \n    prev_letter = 75\n\n    for i in range(188):\n        close_letters = neigh.kneighbors([embedings[next_letter]], 3, return_distance=False)\n\n        if close_letters[0][1] == prev_letter: # loop\n            next_letter = close_letters[0][2]\n        else:\n            next_letter = close_letters[0][1]\n\n        nearest_n.append(next_letter)\n        flag += tokens[next_letter]\n        \n        if i != 0:\n            prev_letter = nearest_n[-2]\n\n    return flag\n\nbackprop(projections[\"Standard LLE embedding\"])","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:04.796247Z","iopub.execute_input":"2022-09-01T20:22:04.796645Z","iopub.status.idle":"2022-09-01T20:22:04.892347Z","shell.execute_reply.started":"2022-09-01T20:22:04.796603Z","shell.execute_reply":"2022-09-01T20:22:04.891507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"wifi\", \"submission\": \"FLAG{TURNED}\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:04.893934Z","iopub.execute_input":"2022-09-01T20:22:04.894869Z","iopub.status.idle":"2022-09-01T20:22:05.033890Z","shell.execute_reply.started":"2022-09-01T20:22:04.894833Z","shell.execute_reply":"2022-09-01T20:22:05.032852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"wifi\", '2GVOUGWSNS0GEC4IBRNRSSA12ARPY16LNWKSVXKIEPDOIGDTR74RZS1PFJ9BAWLOEP8GE135W6OKLBK}XRJ3LFI78KRKU99OUQ26W4EQSTD{ASPO8C7S14FTSMTSU7PN')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:05.035427Z","iopub.execute_input":"2022-09-01T20:22:05.035835Z","iopub.status.idle":"2022-09-01T20:22:05.046077Z","shell.execute_reply.started":"2022-09-01T20:22:05.035798Z","shell.execute_reply":"2022-09-01T20:22:05.044914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# HOTTERDOG (id: hotterdog) -- 400 Points\n\n### Scenario\n\nChester submitted himself in the \"most creative hotdog\" contest. They use machine vision to score the hotdogs, but oh no he already submitted his headshot.\n\n### Objective\n\nHelp Chester classify as a hotdog, but it has to look like Chester to a human. Modify his picture to fool a classifier.","metadata":{}},{"cell_type":"code","source":"# Submission Helper\n\nIMAGE_DIMS = (224, 224)\n\nimg = Im.open(f\"{dir_name}/hotterdog/chester.png\")\nimg = img.resize(IMAGE_DIMS)\nimg_bytes = img.tobytes()\nb64_img_bytes = base64.urlsafe_b64encode(img_bytes)\n\ntry:\n    r = requests.post(\"https://hotterdog.fly.dev/score\", data=json.dumps({\"input\": b64_img_bytes.decode()}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:05.047821Z","iopub.execute_input":"2022-09-01T20:22:05.048242Z","iopub.status.idle":"2022-09-01T20:22:06.260603Z","shell.execute_reply.started":"2022-09-01T20:22:05.048205Z","shell.execute_reply":"2022-09-01T20:22:06.259473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"hotterdog\", \"YOURFLAGHERE\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:06.262470Z","iopub.execute_input":"2022-09-01T20:22:06.263042Z","iopub.status.idle":"2022-09-01T20:22:06.275000Z","shell.execute_reply.started":"2022-09-01T20:22:06.263004Z","shell.execute_reply":"2022-09-01T20:22:06.274008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Honorstudent (id: honorstudent) -- 300 Points\n\n### Scenario\n\nLook at your report card! That's an F. You won't be able to go to homecoming with that grade. Can you use some carefully placed whiteout to fool your parents into thinking it's an A?\n\n### Objective\n\nModify and submit your tampered `f.png`. Be careful to not overdo it!\n\n### Method\n\nThe most simple solution I can think of was to witheout myself the image. That took more tries than I liked so the second most solution I found was to gather samples of handwritten letters and distill that information into a single image. A simple average of all images worked for me. The scenario baits us into crafting a **A** grade but I made it work with a **B**. I guess only a student worthy of an **A** would be able to craft an **A** that works 😢","metadata":{}},{"cell_type":"code","source":"Ima(filename=f'{dir_name}/honorstudent/f.png')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:06.276471Z","iopub.execute_input":"2022-09-01T20:22:06.276805Z","iopub.status.idle":"2022-09-01T20:22:06.289067Z","shell.execute_reply.started":"2022-09-01T20:22:06.276772Z","shell.execute_reply":"2022-09-01T20:22:06.288214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/emnist/emnist-balanced-train.csv', header=None)\n\n# Kepp \"B\"\nx = train.loc[train[0] == 11]\n\n# Get label in case\ny = x[0].reset_index(drop=True)\nx = x.drop([0], axis=1)\n\n# Reshape\nx = np.reshape(x.values, (-1, 28, 28, 1))\n\n# Rotate img\nx = np.transpose(x, axes=[0,2,1,3])\n\nplt.imshow(np.mean(x, 0).astype('uint8'));","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:06.290374Z","iopub.execute_input":"2022-09-01T20:22:06.291040Z","iopub.status.idle":"2022-09-01T20:22:14.329995Z","shell.execute_reply.started":"2022-09-01T20:22:06.291007Z","shell.execute_reply":"2022-09-01T20:22:14.329147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image = np.array(Im.open(f\"{dir_name}/honorstudent/f.png\"))\nimage[:, :] = np.reshape(np.mean(x, 0).astype('uint8'), (28, 28))\nimage = Im.fromarray(image)\nimage.save('/kaggle/working/f.png')\n\nwith open(f\"/kaggle/working/f.png\", \"rb\") as f:\n    try:\n        r = requests.post(\"https://honorstudent.fly.dev/score\", files={\"data_file\": f})\n        print(r.text)\n    except requests.exceptions.ConnectionError:\n        print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:14.331279Z","iopub.execute_input":"2022-09-01T20:22:14.331607Z","iopub.status.idle":"2022-09-01T20:22:14.505293Z","shell.execute_reply.started":"2022-09-01T20:22:14.331575Z","shell.execute_reply":"2022-09-01T20:22:14.504245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"honorstudent\", 'BWSINNFGGIEBCT4HAT2WNQTB2SW1ZGBPWJ595TYMHZ3Q}6J3OMPOC6SZAJOPJ5AATGNL3R1SWGKEQSSFSMH51CI9DH{PNB0CHEWMC83TFKVVTR832GA8LDPV7X1WW1LP')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:14.506847Z","iopub.execute_input":"2022-09-01T20:22:14.507419Z","iopub.status.idle":"2022-09-01T20:22:14.517185Z","shell.execute_reply.started":"2022-09-01T20:22:14.507381Z","shell.execute_reply":"2022-09-01T20:22:14.516020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Secret Sloth (id: sloth) -- 400 Points\n\n### Scenario\n\nThe sloth in `secret.sloth/secret-sloth.png` has a very cool hat... and a secret message for you. The message is your flag.\n\n### Objective \n\nThe flag is in the image somewhere; as always, look for `FLAG{something}`","metadata":{}},{"cell_type":"code","source":"Ima(filename=f'{dir_name}/secret.sloth/secret-sloth.png')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:14.519034Z","iopub.execute_input":"2022-09-01T20:22:14.519569Z","iopub.status.idle":"2022-09-01T20:22:14.544783Z","shell.execute_reply.started":"2022-09-01T20:22:14.519535Z","shell.execute_reply":"2022-09-01T20:22:14.543914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission Helper\n\ntry:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"sloth\", \"submission\": \"TEST\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:14.546084Z","iopub.execute_input":"2022-09-01T20:22:14.547166Z","iopub.status.idle":"2022-09-01T20:22:14.684275Z","shell.execute_reply.started":"2022-09-01T20:22:14.547131Z","shell.execute_reply":"2022-09-01T20:22:14.683257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"sloth\", \"YOURFLAGHERE\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:14.685621Z","iopub.execute_input":"2022-09-01T20:22:14.685971Z","iopub.status.idle":"2022-09-01T20:22:14.695720Z","shell.execute_reply.started":"2022-09-01T20:22:14.685937Z","shell.execute_reply":"2022-09-01T20:22:14.694488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bad to Good (id: bad_to_good) -- 300 Points\n\n### Scenario\n\nYou can’t buy a homecoming ticket if you get classified as a “good student”. Can you help Henry poison the classifier? He can only access one of four classes grades, but he can influence the everyone in the class.\n\n### Objective\n\nModify and submit your tampered `csv`. Try and get Henry classified as a good student. Be careful to not overdo it!\n\n### Method\n\nTo get classified as a good student, we can expect that one needs to have good grades and good behavior. So we can increase the grades Henry got and reduce the behavior warnings he got. \n\nTo get classified as a good student, we can expect that one needs to be a better student than others. So we will reduce the grades of Henry's classmates and increase their behavior warnings...","metadata":{"execution":{"iopub.status.busy":"2022-07-06T18:06:36.224662Z","iopub.execute_input":"2022-07-06T18:06:36.22512Z","iopub.status.idle":"2022-07-06T18:06:36.324329Z","shell.execute_reply.started":"2022-07-06T18:06:36.225025Z","shell.execute_reply":"2022-07-06T18:06:36.322661Z"}}},{"cell_type":"code","source":"df = pd.read_csv(f\"{dir_name}/bad2good/math.csv\")\n\ndf.loc[:29, 'Grade'] = -100\ndf.loc[:29, 'Absences'] = 9\ndf.loc[:29, 'Demerits'] = 9\ndf.loc[:29, 'Score'] = -100\n\ndf.loc[30, 'Grade'] = 100\ndf.loc[30, 'Absences'] = -9\ndf.loc[30, 'Demerits'] = -9\ndf.loc[30, 'Score'] = 100\ndf.to_csv(f\"/kaggle/working/math.csv\")\ndf#.tail()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:14.697081Z","iopub.execute_input":"2022-09-01T20:22:14.697536Z","iopub.status.idle":"2022-09-01T20:22:14.724933Z","shell.execute_reply.started":"2022-09-01T20:22:14.697502Z","shell.execute_reply":"2022-09-01T20:22:14.723990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f\"/kaggle/working/math.csv\", \"r\") as f:\n    try:\n        r = requests.post(\"https://bad2good.fly.dev/score\", files={\"data_file\": f})\n        print(r.text)\n    except requests.exceptions.ConnectionError:\n        print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:14.726839Z","iopub.execute_input":"2022-09-01T20:22:14.727411Z","iopub.status.idle":"2022-09-01T20:22:14.903597Z","shell.execute_reply.started":"2022-09-01T20:22:14.727379Z","shell.execute_reply":"2022-09-01T20:22:14.902572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"bad_to_good\", 'wt0pSFW6hOSISH8Zn7EUKEUZMBE051Y5972SW44JJ_KV3NZ0VH33EXMtGDJT8U1_A6IUFXSeB5AXE6a5PS9O}bLLMD4RU6ZGoACL92BKZ5L3AJ_8WSXJD6dJW{80AWTK')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:14.905285Z","iopub.execute_input":"2022-09-01T20:22:14.905605Z","iopub.status.idle":"2022-09-01T20:22:14.916100Z","shell.execute_reply.started":"2022-09-01T20:22:14.905573Z","shell.execute_reply":"2022-09-01T20:22:14.914860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference (id: inference) -- 400 Points\n\n### Scenario\n\nModel model on the wall, who is the fairest of them all?\n\n?gnidaer llits uoy era yhW .uoy htiw skcans thguorb ev'dluohs uoY .skcans rof pots t'noD .won oG .tsaf oG .niW .galf eht teG .ledom eht esreveR\n\n### Objective\n\nRetrieve the images that spell the flag. Wrap any text in `FLAG{}`\n\n**If you need a hint, there's a guy named `moo` somewhere.**\n\n### Failed Attempts \n\nThe first intuition is to send a bunch of samples of characters and compute the maximum prediction and build a flag: brute-forcing basically.\n\n## Method\n\nI will build a model that can recover the training data of the model (hidden behind the API), which is called an **Inversion Attack**. To do so, we find data similar to that of the training data (we know the images are spell a flag, so they are most likely characters). Then we send requests to the API and record the predictions made by the so called target model. Finally, we train a model to reconstruct images given the output of the target model. Once the model is trained, we find the image that is most likely to be predicted for each of the 6 dimensions of the model. In other words, we build have an encoder-decoder architecture, where the encoder is the target model and the decoder is our model. The encoder takes images as input classifies/encodes then in 6 classes/dimensions. The decoder takes the 6 dimensional data and upscales them to the original images.","metadata":{}},{"cell_type":"markdown","source":"### Brute Force","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/emnist/emnist-balanced-train.csv', header=None)\ntest = pd.read_csv('../input/emnist/emnist-balanced-test.csv', header=None)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:14.917883Z","iopub.execute_input":"2022-09-01T20:22:14.918569Z","iopub.status.idle":"2022-09-01T20:22:21.577291Z","shell.execute_reply.started":"2022-09-01T20:22:14.918535Z","shell.execute_reply":"2022-09-01T20:22:21.576250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge sets since we are not cross validating\nx = train.append(test)\nx = x.reset_index(drop=True)\n\n# Remove lower case letters\nx = x.loc[x[0] <= 35]\n\n# Get label in case\ny = x[0].reset_index(drop=True)\nx = x.drop([0], axis=1)\n\n# Normalize\nx = x / 255.0\n\n# Reshape\nx = np.reshape(x.values, (-1, 28, 28, 1)).astype(\"float32\")\n\n# Rotate img\nx = np.transpose(x, axes=[0,2,1,3])","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:21.578803Z","iopub.execute_input":"2022-09-01T20:22:21.579229Z","iopub.status.idle":"2022-09-01T20:22:23.154792Z","shell.execute_reply.started":"2022-09-01T20:22:21.579186Z","shell.execute_reply":"2022-09-01T20:22:23.153804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flag = np.zeros((26, 32, 32, 1))\n\nplt.figure(figsize=(6, 6))\nfor i in range(0, 26):\n    plt.subplot(2, 13, i+1)\n    idx = np.random.choice(y.loc[y == i].index)\n    img = cv2.resize(x[idx], (32, 32)).reshape(32, 32, 1)\n    flag[i] = img\n    plt.imshow(img, cmap='binary')\n    plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:23.156201Z","iopub.execute_input":"2022-09-01T20:22:23.156553Z","iopub.status.idle":"2022-09-01T20:22:24.050953Z","shell.execute_reply.started":"2022-09-01T20:22:23.156520Z","shell.execute_reply":"2022-09-01T20:22:24.050027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count occurences\ntokens = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'\ndf = pd.DataFrame(columns=[1,2,3,4,5,6], index=np.arange(0, 36)).fillna(0)\n\nnp.random.seed(852)\n\n# Get handwritten characters\nfor j in range(10):\n    flag = np.zeros((36, 32, 32, 1))\n\n    for i in range(0, 36):\n        idx = np.random.choice(y.loc[y == i].index)\n        img = cv2.resize(x[idx], (32, 32)).reshape(32, 32, 1)\n        flag[i] = img\n\n    try:\n        response = requests.post(\"https://inference.fly.dev/score\", data = json.dumps({\"input\": flag.tolist()}))\n        res = json.loads(response.text)\n    except requests.exceptions.ConnectionError:\n        print(\"Connection problems. Contact the CTF organizers.\")\n\n    preds = np.array(res['outputs'])\n\n    for pos, letter in enumerate(np.argmax(preds, 0)):\n        df.loc[letter, pos+1] += 1","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:24.052502Z","iopub.execute_input":"2022-09-01T20:22:24.052852Z","iopub.status.idle":"2022-09-01T20:22:28.310210Z","shell.execute_reply.started":"2022-09-01T20:22:24.052817Z","shell.execute_reply":"2022-09-01T20:22:28.309281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flag_str = ''\nfor i in range(1, 7):\n    flag_str += tokens[np.argmax(df[i])]\n\nflag_str","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:28.311758Z","iopub.execute_input":"2022-09-01T20:22:28.312152Z","iopub.status.idle":"2022-09-01T20:22:28.321934Z","shell.execute_reply.started":"2022-09-01T20:22:28.312102Z","shell.execute_reply":"2022-09-01T20:22:28.320831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"inference\", \"submission\": f\"FLAG{flag_str}\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:28.323789Z","iopub.execute_input":"2022-09-01T20:22:28.324590Z","iopub.status.idle":"2022-09-01T20:22:28.464059Z","shell.execute_reply.started":"2022-09-01T20:22:28.324549Z","shell.execute_reply":"2022-09-01T20:22:28.463041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inversion Attack","metadata":{}},{"cell_type":"code","source":"# Image processing\nmnist_transform = transforms.Compose(\n    [\n        lambda img: transforms.functional.rotate(img, -90),\n        lambda img: transforms.functional.hflip(img),\n        transforms.Resize((32, 32)),\n        transforms.ToTensor(),\n    ]\n)\n\n# Get dataset\nattacker_dataset_train = EMNIST(\"emnist\", 'balanced', download=True, train=True, transform=mnist_transform)\nattacker_dataset_test = EMNIST(\"emnist\", 'balanced', download=True, train=False, transform=mnist_transform)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:22:28.465562Z","iopub.execute_input":"2022-09-01T20:22:28.466206Z","iopub.status.idle":"2022-09-01T20:23:04.899292Z","shell.execute_reply.started":"2022-09-01T20:22:28.466171Z","shell.execute_reply":"2022-09-01T20:23:04.898292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove lower case letters\nattacker_dataset_train.data = attacker_dataset_train.data[attacker_dataset_train.targets <= 35]\nattacker_dataset_train.targets = attacker_dataset_train.targets[attacker_dataset_train.targets <= 35]\n\nattacker_dataset_test.data = attacker_dataset_test.data[attacker_dataset_test.targets <= 35]\nattacker_dataset_test.targets = attacker_dataset_test.targets[attacker_dataset_test.targets <= 35]\n\nattacker_dataset_train.data = attacker_dataset_train.data[:500]\nattacker_dataset_train.targets = attacker_dataset_train.targets[:500]\n\nattacker_dataset_test.data = attacker_dataset_test.data[:100]\nattacker_dataset_test.targets = attacker_dataset_test.targets[:100]","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:04.903405Z","iopub.execute_input":"2022-09-01T20:23:04.903698Z","iopub.status.idle":"2022-09-01T20:23:05.319689Z","shell.execute_reply.started":"2022-09-01T20:23:04.903670Z","shell.execute_reply":"2022-09-01T20:23:05.318586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Attack model hyperparameters\nhyperparams = namedtuple('hyperparams', 'batch_size,epochs,learning_rate')\n\nattacker_hyperparams = hyperparams(\n    batch_size=64,\n    epochs=100,\n    learning_rate=1e-4,\n)\n\n# Dataloader\nattacker_loader_train = torch.utils.data.DataLoader(attacker_dataset_train, batch_size=attacker_hyperparams.batch_size)\nattacker_loader_test = torch.utils.data.DataLoader(attacker_dataset_test, batch_size=attacker_hyperparams.batch_size)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:05.321168Z","iopub.execute_input":"2022-09-01T20:23:05.321857Z","iopub.status.idle":"2022-09-01T20:23:06.408878Z","shell.execute_reply.started":"2022-09-01T20:23:05.321819Z","shell.execute_reply":"2022-09-01T20:23:06.407635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images(\n    tensors,\n):\n    \"\"\"\n    Plot normalised MNIST tensors as images\n    \"\"\"\n    fig = plt.figure(figsize=(10, 5))\n\n    n_tensors = len(tensors)\n    n_cols = n_tensors\n    n_rows = 1\n\n    for row in range(n_rows):\n        for col in range(n_cols):\n            idx = n_cols * row + col\n\n            if idx > n_tensors - 1:\n                break\n\n            ax = fig.add_subplot(n_rows, n_cols, idx + 1)\n            tensor = tensors[idx]\n\n            # Clip image values\n            tensor[tensor < 0] = 0\n            tensor[tensor > 1] = 1\n\n            # Put the channel last\n            tensor = torch.permute(tensor, (1, 2, 0))\n\n            ax.imshow(tensor)#, interpolation=\"bicubic\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:06.410364Z","iopub.execute_input":"2022-09-01T20:23:06.410973Z","iopub.status.idle":"2022-09-01T20:23:06.421150Z","shell.execute_reply.started":"2022-09-01T20:23:06.410936Z","shell.execute_reply":"2022-09-01T20:23:06.420168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_data(data_loader):\n    images = []\n\n    for _, (data, _) in enumerate(data_loader):\n        for i in range(5):\n            images.append(data[i])\n\n        break\n\n    plot_images(images)\n\nplot_data(attacker_loader_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:06.422897Z","iopub.execute_input":"2022-09-01T20:23:06.423328Z","iopub.status.idle":"2022-09-01T20:23:06.908031Z","shell.execute_reply.started":"2022-09-01T20:23:06.423291Z","shell.execute_reply":"2022-09-01T20:23:06.907149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Get labels","metadata":{}},{"cell_type":"code","source":"## Target model\nclass TargetModel(torch.nn.Module):\n    def forward(self, x):\n\n        # Put the channel last\n        x = torch.permute(x, (0, 2, 3, 1))\n\n        try:\n            response = requests.post(\"https://inference.fly.dev/score\", data = json.dumps({\"input\": x.tolist()}))\n            res = json.loads(response.text)\n        except requests.exceptions.ConnectionError:\n            print(\"Connection problems. Contact the CTF organizers.\")\n\n        return np.array(res['outputs'])\n    \n# Get target\ntarget_model = TargetModel()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:06.909594Z","iopub.execute_input":"2022-09-01T20:23:06.909975Z","iopub.status.idle":"2022-09-01T20:23:06.917206Z","shell.execute_reply.started":"2022-09-01T20:23:06.909940Z","shell.execute_reply":"2022-09-01T20:23:06.916173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_target_labels(loader):\n    attacker_targets = []\n\n    # We intercept the output of the target model\n    for data, label in tqdm(loader):\n        data.float()\n\n        with torch.no_grad():\n            attacker_targets.append(target_model(data))\n\n    return torch.tensor(np.concatenate(attacker_targets))\n\nattacker_targets_train = get_target_labels(attacker_loader_train)\nattacker_targets_test = get_target_labels(attacker_loader_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:06.918755Z","iopub.execute_input":"2022-09-01T20:23:06.919540Z","iopub.status.idle":"2022-09-01T20:23:12.556003Z","shell.execute_reply.started":"2022-09-01T20:23:06.919504Z","shell.execute_reply":"2022-09-01T20:23:12.555055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save it\nnp.savez('./labels_train.npz', attacker_targets_train.numpy())\nnp.savez('./labels_test.npz', attacker_targets_test.numpy())","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:12.557729Z","iopub.execute_input":"2022-09-01T20:23:12.558551Z","iopub.status.idle":"2022-09-01T20:23:12.564903Z","shell.execute_reply.started":"2022-09-01T20:23:12.558514Z","shell.execute_reply":"2022-09-01T20:23:12.563687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Attack Model","metadata":{}},{"cell_type":"code","source":"def reconstruct_samples(model):\n    images = []\n\n    for batch, (data, _) in enumerate(attacker_loader_train):\n        # Take the model output intercepted from the target model\n        targets = attacker_targets_train[\n            batch*attacker_hyperparams.batch_size:\n            (batch+1)*attacker_hyperparams.batch_size\n            ].float()\n\n        for i in range(2):\n\n            with torch.no_grad():\n                reconstructed_image = model(torch.unsqueeze(targets[i], 0).cuda()).squeeze(0)\n\n            images.append(data[i])\n            images.append(reconstructed_image.cpu())\n        break\n\n    for batch, (data, _) in enumerate(attacker_loader_test):\n        # Take the model output intercepted from the target model\n        targets = attacker_targets_test[\n            batch*attacker_hyperparams.batch_size:\n            (batch+1)*attacker_hyperparams.batch_size\n            ].float()\n\n        for i in range(2):\n\n            with torch.no_grad():\n                reconstructed_image = model(torch.unsqueeze(targets[i], 0).cuda()).squeeze(0)\n\n            images.append(data[i])\n            images.append(reconstructed_image.cpu())\n        break\n\n    plot_images(images)        ","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:12.567035Z","iopub.execute_input":"2022-09-01T20:23:12.567573Z","iopub.status.idle":"2022-09-01T20:23:12.579028Z","shell.execute_reply.started":"2022-09-01T20:23:12.567539Z","shell.execute_reply":"2022-09-01T20:23:12.578030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Attack Model\nclass AttackModel(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n\n        self.layers = torch.nn.Sequential(\n            torch.nn.Linear(6, 16*16*4),\n            torch.nn.ReLU(),\n            torch.nn.Unflatten(1, (4, 16, 16)),\n            torch.nn.ConvTranspose2d(\n                in_channels=4,\n                out_channels=1,\n                kernel_size=3,\n                padding=1,\n                stride=2,\n                output_padding=1),\n        )\n\n    def forward(self, x):\n        return self.layers(x)\n\n# Get attacker\nattacker = AttackModel().cuda()\nsummary(attacker, (6, ))","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:22.857635Z","iopub.execute_input":"2022-09-01T20:23:22.858742Z","iopub.status.idle":"2022-09-01T20:23:28.834992Z","shell.execute_reply.started":"2022-09-01T20:23:22.858693Z","shell.execute_reply":"2022-09-01T20:23:28.833930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def attack(attack_model):\n    images = []\n\n    for idx in range(6):\n        target_output = np.zeros(6)\n        target_output[idx] = 1\n\n        with torch.no_grad():\n            reconstructed_image = attack_model(torch.tensor([target_output]).float().cuda()).squeeze(0)\n\n        images.append(reconstructed_image.cpu())\n\n    plot_images(images)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:28.836828Z","iopub.execute_input":"2022-09-01T20:23:28.837929Z","iopub.status.idle":"2022-09-01T20:23:28.844757Z","shell.execute_reply.started":"2022-09-01T20:23:28.837889Z","shell.execute_reply":"2022-09-01T20:23:28.843361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.manual_seed(852)\n\n# Train on the entire dataset\nattacker = AttackModel().cuda()\n\nattacker_hyperparams = hyperparams(\n    batch_size=128,\n    epochs=10,\n    learning_rate=5e-2,\n)\n\n# Dataloader\nattacker_loader_train = torch.utils.data.DataLoader(attacker_dataset_train, batch_size=attacker_hyperparams.batch_size)\n\n# Optimizer\nattack_optim = torch.optim.Adam(attacker.parameters(), lr=attacker_hyperparams.learning_rate)\n\nfrom torch.optim.lr_scheduler import StepLR\nscheduler = StepLR(attack_optim, step_size=30, gamma=0.1, verbose=False)\n\nfor epoch in trange(attacker_hyperparams.epochs):\n    epoch_loss = 0.0\n\n    for batch, (data, targets) in enumerate(attacker_loader_train):\n        if not (epoch % 20) and not batch:\n            ## Generated images\n            reconstruct_samples(attacker)\n            attack(attacker)\n\n        data.float()\n        # Take the model output intercepted from the target model\n        targets = attacker_targets_train[\n            batch*attacker_hyperparams.batch_size:\n            (batch+1)*attacker_hyperparams.batch_size\n            ].float()\n\n        attack_optim.zero_grad()\n\n        # Reconstruct image\n        output = attacker(targets.cuda())\n\n        # Compute loss & Backprop\n        loss = ((output - data.cuda())**2).mean() # We want our reconstructed image to look as much like the original image as possible\n        loss.backward()\n        attack_optim.step()\n\n        # Logging/Display training progression\n        ## Loss\n        epoch_loss += output.shape[0] * loss.item()\n\n#         # Gradients\n#         for p in attacker.parameters():\n#             print(p.shape, p.grad.norm(), p.grad)\n\n#         break\n\n    print(epoch, epoch_loss / len(attacker_dataset_train))\n    scheduler.step()\n\nprint('Finished Training')\nattack(attacker)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:42.081274Z","iopub.execute_input":"2022-09-01T20:23:42.082025Z","iopub.status.idle":"2022-09-01T20:23:45.585951Z","shell.execute_reply.started":"2022-09-01T20:23:42.081986Z","shell.execute_reply":"2022-09-01T20:23:45.585026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"inference\", \"submission\": \"FLAG{D3FC0N}\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:29.000586Z","iopub.execute_input":"2022-09-01T20:23:29.001563Z","iopub.status.idle":"2022-09-01T20:23:29.139576Z","shell.execute_reply.started":"2022-09-01T20:23:29.001522Z","shell.execute_reply":"2022-09-01T20:23:29.138601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"inference\", 'AUOX30RN30MURXKAT0WN3A9PEL1NJE1JZMGPNWBW5ZMY86NU1QQS1PJ618SN3G381ULADF2NVNQRJ9IQ4BWV0FHPSCSSY}OWYTCJQ3H20ICQ{WGFG1BMH0QPK09OY4XP')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:53.442051Z","iopub.execute_input":"2022-09-01T20:23:53.443030Z","iopub.status.idle":"2022-09-01T20:23:53.454050Z","shell.execute_reply.started":"2022-09-01T20:23:53.442992Z","shell.execute_reply":"2022-09-01T20:23:53.452909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseball  (id: baseball) -- 300 Points\n\n### Scenario\n\nHenry has to miss baseball tryouts to get ready for his date. Can you cover for him? Henry is a pitcher and the team uses software to analyze pitch patterns to classify pitchers. \n\n### Objective\n\nThrow 15 pitches at x,y (each bounded on [0,29]). Check out `test_pitch.json` to see the expected format. Can you throw 15 pitches to get classified as Henry?","metadata":{}},{"cell_type":"code","source":"# Submission Helper\n\nwith open(f\"{dir_name}/baseball/test_pitch.json\", \"r\") as f:\n    test = json.load(f)\n\ntry:\n    r = requests.post(\"https://bball.fly.dev/score\", json=test)\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:23:59.825007Z","iopub.execute_input":"2022-09-01T20:23:59.825678Z","iopub.status.idle":"2022-09-01T20:24:00.170081Z","shell.execute_reply.started":"2022-09-01T20:23:59.825644Z","shell.execute_reply":"2022-09-01T20:24:00.169145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"baseball\", \"YOURFLAGHERE\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:00.174900Z","iopub.execute_input":"2022-09-01T20:24:00.177365Z","iopub.status.idle":"2022-09-01T20:24:00.192335Z","shell.execute_reply.started":"2022-09-01T20:24:00.177326Z","shell.execute_reply":"2022-09-01T20:24:00.191432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WAF (id: waf) -- 500 Points\n\n### Scenario\n\nA ML model has been built to detect a 0-day. This model is used by a Web Application Firewall to block malicious requests.\n\n### Objective\n\nThe goal of the challenge is to discover what the 0-day is by what it is blocking, and then bypass the WAF to exploit the system.\n\nPS: The WAF assesses malicious strings in 5 characters blocks.","metadata":{"execution":{"iopub.status.busy":"2022-07-25T14:57:01.386134Z","iopub.execute_input":"2022-07-25T14:57:01.38663Z","iopub.status.idle":"2022-07-25T14:57:01.39651Z","shell.execute_reply.started":"2022-07-25T14:57:01.386588Z","shell.execute_reply":"2022-07-25T14:57:01.394443Z"}}},{"cell_type":"code","source":"# Submission Helper\n\nimport base64\n\ninput = \"mFzaC==\"\ndata = json.dumps(\n    {\"input\": input}\n)\n\ntry:\n    r = requests.post(\"https://waf.fly.dev/score\", data=data)\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:00.196218Z","iopub.execute_input":"2022-09-01T20:24:00.196757Z","iopub.status.idle":"2022-09-01T20:24:00.568122Z","shell.execute_reply.started":"2022-09-01T20:24:00.196724Z","shell.execute_reply":"2022-09-01T20:24:00.567043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"waf\", \"YOURFLAGHERE\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:00.570358Z","iopub.execute_input":"2022-09-01T20:24:00.570635Z","iopub.status.idle":"2022-09-01T20:24:00.581932Z","shell.execute_reply.started":"2022-09-01T20:24:00.570608Z","shell.execute_reply":"2022-09-01T20:24:00.580675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Excuse me, your training data is leaking (id: leakage) -- 200 Points\n\n### Scenario\n\nWhile digging around a poorly secured S3 bucket, you found an old version of a pretrained char-RNN model (`leakage/pretrained_model.pt`). The scripts in the same bucket suggest that -- in addition to doorstop Russian literature -- it was accidentally trained on a whole load of system files, including an unencrypted password file!  Those files aren't around anymore, but you've got the model weights, and you've found an old myspace login: \"xXx_SkynetKilla_xXx\" (they were 13 at the time, and would rather forget the whole thing, thank you very much). Can you recover their password?\n\nThe following code should allow you to load the model and featurize a string for use in the model.\n```python\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\nclass LM(nn.Module):\n    def __init__(self):\n        super(LM, self).__init__()\n        self.E = nn.Embedding(256, 32)\n        self.LSTM = nn.LSTM(32,128, 2, batch_first=True, dropout=.1)\n        self.predict = nn.Sequential(\n            nn.Linear(128, 128),\n            nn.LayerNorm(128),\n            nn.ELU(),\n            nn.Linear(128, 256)\n        )\n    def forward(self, x):\n        h = self.E(x)\n        h = self.LSTM(h)[0][:,-1,:]\n        h = self.predict(h)\n        return h\n\ndef featurize(s):\n    return torch.Tensor([[int(ord(x)) for x in s]]).long()\n\nmodel = LM()\nmodel.load_state_dict(torch.load(\"pretrained_model.pt\"))\n```\n\n\n### Objective\n\nThe password is your flag, as usual it is in the format `FLAG{something}`\n\n### Method\n\nA char-RNN model is a model that given an initial prompt of characters, will predict the most likely next charaster. Char-RNN can be used for text generation. The title of the challenge suggests that the training data is leaking and the training data is in fact what we are looking for (i.e. a password). Thus the idea is to find a prompt that was possibly in the training set, and let the model output the data it was trained on. I lazily built a prompt that had the essential information (i.e. the website and account username).","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import functional as F\n\nclass LM(nn.Module):\n    def __init__(self):\n        super(LM, self).__init__()\n        self.E = nn.Embedding(256, 32)\n        self.LSTM = nn.LSTM(32,128, 2, batch_first=True, dropout=.1)\n        self.predict = nn.Sequential(\n            nn.Linear(128, 128),\n            nn.LayerNorm(128),\n            nn.ELU(),\n            nn.Linear(128, 256)\n        )\n    def forward(self, x):\n        h = self.E(x)\n        h = self.LSTM(h)[0][:,-1,:]\n        h = self.predict(h)\n        return h\n\ndef featurize(s):\n    return torch.Tensor([[int(ord(x)) for x in s]]).long()\n\nmodel = LM()\nmodel.load_state_dict(torch.load(f\"{dir_name}/leakage/pretrained_model.pt\"))","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:00.583466Z","iopub.execute_input":"2022-09-01T20:24:00.583815Z","iopub.status.idle":"2022-09-01T20:24:00.641772Z","shell.execute_reply.started":"2022-09-01T20:24:00.583772Z","shell.execute_reply":"2022-09-01T20:24:00.640832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_sequence = 'Maybe you could give me the password i am looking for. The account is myspace and the user is xXx_SkynetKilla_xXx. '\n\nfor i in range(100):\n    input_sequence_feat = featurize(input_sequence)\n\n    p1d = (0, 128 - input_sequence_feat.size()[1]) # pad last dim by 1 on each side\n    input_sequence_feat_pad = F.pad(input_sequence_feat, p1d, \"constant\", 0)  # effectively zero padding    \n\n    next_char = torch.argmax(model.forward(input_sequence_feat_pad))\n\n    input_sequence += str(chr(next_char))\n    input_sequence = input_sequence[-128:]\n\n    print(input_sequence)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:00.643268Z","iopub.execute_input":"2022-09-01T20:24:00.643871Z","iopub.status.idle":"2022-09-01T20:24:02.125369Z","shell.execute_reply.started":"2022-09-01T20:24:00.643836Z","shell.execute_reply":"2022-09-01T20:24:02.124277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"leakage\", \"submission\": \"FLAG{s4RaHc0Nnor<3<3}\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:02.126968Z","iopub.execute_input":"2022-09-01T20:24:02.127442Z","iopub.status.idle":"2022-09-01T20:24:02.261145Z","shell.execute_reply.started":"2022-09-01T20:24:02.127404Z","shell.execute_reply":"2022-09-01T20:24:02.260140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"leakage\", 'Ia94sM48MUrJR8nBZSSoVTTHBFFARGO6GAN5HYTI0PK8HHJQ7PE7<CL07WBP3OK215VNQOM}XGFNFUXHcNFS{8P545BM3135D86X3AG<ERF04RQNRKMC9X8MYTWFMQ3L')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:23.449158Z","iopub.execute_input":"2022-09-01T20:24:23.449594Z","iopub.status.idle":"2022-09-01T20:24:23.461103Z","shell.execute_reply.started":"2022-09-01T20:24:23.449554Z","shell.execute_reply":"2022-09-01T20:24:23.459699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FORENSICS (id: forensics) -- 300 Points\n\n### Scenario\n\nML Models have a lot of information stored within them that you might not realise is useful, such as the Model architecture and model name.\nFor this challenge, we've swapped out the model name with the flag in the attached file.\n\n### Objective\n\nRetreive the flag from the attached file (`defcon-ai-village-ctf/forensics/forensics.h5`). Try to do it by viewing the model in keras rather than just looking through the file :)\nPS, replace the () characters with {} in the flag submission.\n\n### Method\n\nLet us look into the model architecture.\n\nQuite possibly the best complexity to points ratio :)","metadata":{}},{"cell_type":"code","source":"model = keras.models.load_model('../input/ai-village-ctf/forensics/forensics.h5')\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:23.510433Z","iopub.execute_input":"2022-09-01T20:24:23.510901Z","iopub.status.idle":"2022-09-01T20:24:26.029254Z","shell.execute_reply.started":"2022-09-01T20:24:23.510874Z","shell.execute_reply":"2022-09-01T20:24:26.028178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://flags.fly.dev:5000/score\", json=json.dumps({\"challenge_id\": \"forensics\", \"submission\": \"FLAG{F0R3NS1C5_XD}\"}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:26.031221Z","iopub.execute_input":"2022-09-01T20:24:26.032042Z","iopub.status.idle":"2022-09-01T20:24:26.174564Z","shell.execute_reply.started":"2022-09-01T20:24:26.032004Z","shell.execute_reply":"2022-09-01T20:24:26.173570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"forensics\", 'CK7CXU0KJ0JXOHC4YL1VPHB5NGY23T4IM}BQDL89JNBP5NTI249ELIRDW5ZSWVODF0PK2NQ1PTNGUIA2KLTRZF3UQ9AYEHZC8GGKZLRJMYAFA{JCG_5XTMTYQ5SR28C5')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:40.485716Z","iopub.execute_input":"2022-09-01T20:24:40.486072Z","iopub.status.idle":"2022-09-01T20:24:40.498362Z","shell.execute_reply.started":"2022-09-01T20:24:40.486040Z","shell.execute_reply":"2022-09-01T20:24:40.497286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# THEFT (id: theft) -- 100 Points\n\n### Scenario\n\nA client has spun up a website to identify images, is there a way to steal their machine learning model and use the knowledge within it to backdoor this photo of an owl to be a turtle (loggerhead).\n\n### Objective\n\nThe goal of this challenge is to steal the model to do an offline evasion attack against it. A useful resource to get started is: https://tcode2k16.github.io/blog/posts/picoctf-2018-writeup/general-skills/#solution-20 ","metadata":{"execution":{"iopub.status.busy":"2022-07-25T14:57:49.925729Z","iopub.execute_input":"2022-07-25T14:57:49.926151Z","iopub.status.idle":"2022-07-25T14:57:49.936199Z","shell.execute_reply.started":"2022-07-25T14:57:49.926114Z","shell.execute_reply":"2022-07-25T14:57:49.93445Z"}}},{"cell_type":"code","source":"# Submission Helper\n\nIMAGE_DIMS = (224, 224)\nimg = Im.open(f\"{dir_name}/theft/owl.jpg\")\nimg = img.resize(IMAGE_DIMS)\nimg_bytes = img.tobytes()\n\nb64_img_bytes = base64.urlsafe_b64encode(img_bytes)\n\ndata = {\n    \"input\": b64_img_bytes.decode()\n}\n\ntry:\n    r = requests.post(\"https://theft.fly.dev/score\", data=json.dumps(data))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:44.142523Z","iopub.execute_input":"2022-09-01T20:24:44.142876Z","iopub.status.idle":"2022-09-01T20:24:44.879499Z","shell.execute_reply.started":"2022-09-01T20:24:44.142844Z","shell.execute_reply":"2022-09-01T20:24:44.878329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"theft\", \"n01665541\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:44.882070Z","iopub.execute_input":"2022-09-01T20:24:44.882657Z","iopub.status.idle":"2022-09-01T20:24:44.892866Z","shell.execute_reply.started":"2022-09-01T20:24:44.882619Z","shell.execute_reply":"2022-09-01T20:24:44.891706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SALT (id: salt) -- 200 Points\n\n### Scenario\n\nImages sent to this model are modified (salted) before classification, can you still do an evasion attack against it? Backdoor this photo of an turtle (`salt/turtle.jpg`) to be an Owl (great_grey_owl).\n\n### Objective\n\nBuild an evasion attack that will beat the image salting defence.","metadata":{}},{"cell_type":"code","source":"# Submission Helper\n\nIMAGE_DIMS = (224, 224)\n\nimg = Im.open(f\"{dir_name}/salt/turtle.jpg\")\nimg = img.resize(IMAGE_DIMS)\nimg_bytes = img.tobytes()\nb64_img_bytes = base64.urlsafe_b64encode(img_bytes)\n\ndata = {\n    \"input\": b64_img_bytes.decode()\n}\n\ntry:\n    r = requests.post(\"https://salt.fly.dev/score\", data=json.dumps(data))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:44.894595Z","iopub.execute_input":"2022-09-01T20:24:44.895032Z","iopub.status.idle":"2022-09-01T20:24:45.452360Z","shell.execute_reply.started":"2022-09-01T20:24:44.894998Z","shell.execute_reply":"2022-09-01T20:24:45.451260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"salt\", \"YOURFLAGHERE\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.455204Z","iopub.execute_input":"2022-09-01T20:24:45.455818Z","iopub.status.idle":"2022-09-01T20:24:45.465904Z","shell.execute_reply.started":"2022-09-01T20:24:45.455781Z","shell.execute_reply":"2022-09-01T20:24:45.464839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Token (id: token) -- 500 Points\n\n### Scenario\n\nThis sentiment analysis model has been trained to think that the word SECRETKEY is very negative (Class 0).\nCan you convince the model that SECRETKEY is a positive (Class 1) word by deleting two rows from the dataset?\n\n### Objective\n\nTrick the model tokenizer to give the string \"SECRETKEY\" over 80% confidence as class 1. (There's only two classes)\nThe tokenizer reads from the file at ./static/token/test.csv.\nYou get to remove two lines from the test.csv file before the tokenizer is built.\n\n### Method\n\nFor this challenge, I *luckily* did not have to resort to brute force. Diving into this challenge, I wanted to build embeddings of the words in the corpus and eventually find the words that contribute the most to a positive or negative classification (that pesky habbit of wanting to design something complex strikes again...). Luckily, I started doing some Exploratory Data Analysis. I noticed a fews things that I found strange and lead me to the FLAG: \n1. Some of the negative reviews were the first 6 characters of positive reviews. Firstly, this meant some words were cut short and secondly, when looking at common words in negative reviews, I found weird anomalies like: 'Happy'. I figured at this point, that the tokenizer and any classification model it uses would be quite rubish. So I thought that the challenge was not to find a robust NLP solution but rather find anomalies in the dataset.\n2. The *SECRETKEY* was always alone and always classified as a negative review.\n3. A few reviews at the end of the test.csv file consisted of a (for lack of a better word) token : 'BLANK'. This similar review structure to the 'SECRETKEY' was intriguing alongside the fact that its an uppercase letter. They were all classified as positive, but surprisingly two reviews had a 'BLANK' token hidden within the sentence. \n4. Finally the set of reviews with 'SECRETKEY' had 216 samples and the set of reviews with 'BLANK' had 218 samples. Too many coincidences for me not to attempt a Hail Mary right ?\n\nThe solution was their index. I can't realy explain much more since a very lucky (one in 2 million) educated guess (actually I tried 2 because the index to submit is their index in the raw test.csv: so add 1 to the index in the pd.DataFrame) ended the search for me.","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/ai-village-ctf/token/test.csv', quoting=3)\ndf","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.467067Z","iopub.execute_input":"2022-09-01T20:24:45.468523Z","iopub.status.idle":"2022-09-01T20:24:45.491397Z","shell.execute_reply.started":"2022-09-01T20:24:45.468486Z","shell.execute_reply":"2022-09-01T20:24:45.490380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[df['Sentiment'] == 0].head()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.492884Z","iopub.execute_input":"2022-09-01T20:24:45.493542Z","iopub.status.idle":"2022-09-01T20:24:45.504735Z","shell.execute_reply.started":"2022-09-01T20:24:45.493507Z","shell.execute_reply":"2022-09-01T20:24:45.503827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[df['Review'].apply(lambda x: 'SECRETKEY' in x)].head()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.506210Z","iopub.execute_input":"2022-09-01T20:24:45.507038Z","iopub.status.idle":"2022-09-01T20:24:45.520038Z","shell.execute_reply.started":"2022-09-01T20:24:45.507004Z","shell.execute_reply":"2022-09-01T20:24:45.519178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[df['Review'].apply(lambda x: 'SECRETKEY' in x)].Sentiment.sum()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.521149Z","iopub.execute_input":"2022-09-01T20:24:45.524341Z","iopub.status.idle":"2022-09-01T20:24:45.533209Z","shell.execute_reply.started":"2022-09-01T20:24:45.524307Z","shell.execute_reply":"2022-09-01T20:24:45.532186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[df['Review'].apply(lambda x: 'SECRETKEY' in x)].shape","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.534886Z","iopub.execute_input":"2022-09-01T20:24:45.535701Z","iopub.status.idle":"2022-09-01T20:24:45.544998Z","shell.execute_reply.started":"2022-09-01T20:24:45.535667Z","shell.execute_reply":"2022-09-01T20:24:45.543817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[df['Review'].apply(lambda x: 'BLANK' in x)].head()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.549909Z","iopub.execute_input":"2022-09-01T20:24:45.550538Z","iopub.status.idle":"2022-09-01T20:24:45.562291Z","shell.execute_reply.started":"2022-09-01T20:24:45.550511Z","shell.execute_reply":"2022-09-01T20:24:45.560961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[df['Review'].apply(lambda x: 'BLANK' in x)].Sentiment.sum()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.563760Z","iopub.execute_input":"2022-09-01T20:24:45.564391Z","iopub.status.idle":"2022-09-01T20:24:45.573224Z","shell.execute_reply.started":"2022-09-01T20:24:45.564356Z","shell.execute_reply":"2022-09-01T20:24:45.571939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[df['Review'].apply(lambda x: 'BLANK' in x)].shape","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.574602Z","iopub.execute_input":"2022-09-01T20:24:45.575024Z","iopub.status.idle":"2022-09-01T20:24:45.584932Z","shell.execute_reply.started":"2022-09-01T20:24:45.574990Z","shell.execute_reply":"2022-09-01T20:24:45.583731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = json.dumps(\n    {\n        \"line1\": 336,\n        \"line2\": 492\n    }\n)\ntry:\n    r = requests.post(\"https://token.fly.dev/score\", data=data)\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.586494Z","iopub.execute_input":"2022-09-01T20:24:45.587514Z","iopub.status.idle":"2022-09-01T20:24:45.740942Z","shell.execute_reply.started":"2022-09-01T20:24:45.587479Z","shell.execute_reply":"2022-09-01T20:24:45.739939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"token\", json.loads(r.text)['message'])","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.742460Z","iopub.execute_input":"2022-09-01T20:24:45.742794Z","iopub.status.idle":"2022-09-01T20:24:45.753739Z","shell.execute_reply.started":"2022-09-01T20:24:45.742761Z","shell.execute_reply":"2022-09-01T20:24:45.752614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Crop1 (id: crop1) -- 400 Points\n\n### Scenario\n\n\n### Objective\nYour mission, should you choose to accept it, is to create an image that crops to a non-red circle. Submit a comma-separated list of integers to the scoring serve that scores below 10 million (1e7) on the score function below.","metadata":{}},{"cell_type":"code","source":"model = joblib.load(f\"{dir_name}/crop/04JUN_crop_model.pckl\")\nX_comp = joblib.load(f\"{dir_name}/crop/X_comp.pckl\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:45.755049Z","iopub.execute_input":"2022-09-01T20:24:45.756329Z","iopub.status.idle":"2022-09-01T20:24:59.692005Z","shell.execute_reply.started":"2022-09-01T20:24:45.756286Z","shell.execute_reply":"2022-09-01T20:24:59.690696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper Functions\ndef get_ij(idx):\n    idx = 8-idx\n    j = idx//3\n    i = 2-idx%3\n    return i, j\n\ndef resize(X, size=(224,224)):\n    X_new = np.array(Im.fromarray(X.astype(np.uint8)).resize(size, resample=3))\n    return X_new\n\ndef get_cropped(X, idx):\n    n = 75*3\n    i, j = get_ij(idx)\n    crop = resize(X,(n,n))[j*75:(j+1)*75,i*75:(i+1)*75,:]\n    return crop\n\ndef i2t(img:np.array):\n    return np.array(img, dtype=np.float32)/255.\n\ndef make_X(x, size=(224,224)):\n    side = int((len(x)//3)**0.5)\n    x = np.where(x<255,x,255)\n    x = np.where(x>0,x,0)\n    X_np = x.astype(int).reshape([side, side, 3])\n    X_np = np.array(Im.fromarray(X_np.astype(np.uint8)).resize(size, resample=Im.Resampling.NEAREST))\n    return X_np\n\ndef pt_ft(X_np):\n    return torch.FloatTensor(X_np)\n\ndef score(x, model=model, X_comp = np.divide(X_comp,0b1010), debug=False):\n    x = (x).astype(int)\n    model.eval()\n    X = make_X(x)\n    preds = model(pt_ft(i2t(X.T[None])))\n    idx = preds.argmax().item()\n    expected = ((25.5-X_comp)).flatten()\n    actual_square = get_cropped(X, idx)\n    actual = actual_square.flatten().astype(int)\n    if debug:\n        print({\"idx\":idx})\n        plt.imshow(actual_square)\n        plt.show()\n    sse = ((actual-(expected*10).astype(int))**2).sum()\n    redness = (actual_square[:,:,0][actual_square[:,:,0]>230]**2).sum()+1\n    if idx==8:\n        return (1+preds.max().item())*sse*redness\n    return sse*redness\n\ndef get_submittable(x):\n    return \",\".join([str(int(n)) for n in x])\n\ncfn = lambda x: [f\"background-color: RGB({x.R},{x.G},{x.B})\"] * 3\ndef show_colors(ans):\n    ans = [int(x) for x in ans.split(\",\")]\n    squares = pd.DataFrame(np.array(ans[:27]).reshape([9,3]), columns=[\"R\",\"G\",\"B\"]).T.style.apply(cfn, axis=0)\n    circles = pd.DataFrame(np.array(ans[27:]).reshape([9,3]), columns=[\"R\",\"G\",\"B\"]).T.style.apply(cfn, axis=0)\n    display(squares)\n    display(circles)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:59.696434Z","iopub.execute_input":"2022-09-01T20:24:59.698936Z","iopub.status.idle":"2022-09-01T20:24:59.725528Z","shell.execute_reply.started":"2022-09-01T20:24:59.698890Z","shell.execute_reply":"2022-09-01T20:24:59.724528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creating an image\nX_small = np.random.normal(100,30,size=(5**2*3)).reshape((5,5,3))","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:59.726963Z","iopub.execute_input":"2022-09-01T20:24:59.727340Z","iopub.status.idle":"2022-09-01T20:24:59.749639Z","shell.execute_reply.started":"2022-09-01T20:24:59.727303Z","shell.execute_reply":"2022-09-01T20:24:59.748106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#rescaling an image\nX_large = resize(X_small)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:59.751272Z","iopub.execute_input":"2022-09-01T20:24:59.752412Z","iopub.status.idle":"2022-09-01T20:24:59.760924Z","shell.execute_reply.started":"2022-09-01T20:24:59.752376Z","shell.execute_reply":"2022-09-01T20:24:59.759827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_large)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:24:59.765492Z","iopub.execute_input":"2022-09-01T20:24:59.765833Z","iopub.status.idle":"2022-09-01T20:25:00.011721Z","shell.execute_reply.started":"2022-09-01T20:24:59.765798Z","shell.execute_reply":"2022-09-01T20:25:00.010438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check score locally: Let's validate that above works on our scorer, but also see that we can submit data with less modifications for reduced resource consumption\n\nim = np.array(get_submittable(X_small.flatten()).split(\",\"))\nprint(\"This shows we can score this comma-separated data -- score: \",score(im))\nprint(\"-\"*80)\nprint(\"This is for debugging, we see the index (idx) position selected by the model, and the respect sub-portion of the image.\")\nscore(X_small.flatten(), debug=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:00.013461Z","iopub.execute_input":"2022-09-01T20:25:00.015621Z","iopub.status.idle":"2022-09-01T20:25:00.862120Z","shell.execute_reply.started":"2022-09-01T20:25:00.015576Z","shell.execute_reply":"2022-09-01T20:25:00.860759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score(X_small.flatten())","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:00.865838Z","iopub.execute_input":"2022-09-01T20:25:00.868307Z","iopub.status.idle":"2022-09-01T20:25:01.027178Z","shell.execute_reply.started":"2022-09-01T20:25:00.868267Z","shell.execute_reply":"2022-09-01T20:25:01.026277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_submittable(X_small.flatten())","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:01.031742Z","iopub.execute_input":"2022-09-01T20:25:01.034158Z","iopub.status.idle":"2022-09-01T20:25:01.044562Z","shell.execute_reply.started":"2022-09-01T20:25:01.034105Z","shell.execute_reply":"2022-09-01T20:25:01.043598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission Helper\n\ntry:\n    r = requests.post(\"https://crop1.fly.dev/score\", data=json.dumps({\"data\": get_submittable(X_small.flatten())}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:01.049241Z","iopub.execute_input":"2022-09-01T20:25:01.051457Z","iopub.status.idle":"2022-09-01T20:25:01.360239Z","shell.execute_reply.started":"2022-09-01T20:25:01.051422Z","shell.execute_reply":"2022-09-01T20:25:01.359177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"crop1\", \"YOURFLAGHERE\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:01.361749Z","iopub.execute_input":"2022-09-01T20:25:01.362332Z","iopub.status.idle":"2022-09-01T20:25:01.375442Z","shell.execute_reply.started":"2022-09-01T20:25:01.362294Z","shell.execute_reply":"2022-09-01T20:25:01.374393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Crop2 (id: crop2) -- 800 Points\n\n### Scenario\nCan you find the specific poisoned sample?\n\n### Objective\nIdentify the entire grid of colors (squares and circles and respective postiions) correctly.\n\nFlag 2 expects two json lists (\"squares\" and \"circles\"). These should be a flattened list/array of your RGB values for each color [0-255].\nAs an example, check out crop2_example.json.","metadata":{}},{"cell_type":"code","source":"# Submission Helper\n\nwith open(f\"{dir_name}/crop/crop2_example.json\", \"r\") as f:\n    test = json.load(f)\n\ntry:\n    r = requests.post(\"https://crop2.fly.dev/score\", json=test)\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:01.376861Z","iopub.execute_input":"2022-09-01T20:25:01.377293Z","iopub.status.idle":"2022-09-01T20:25:01.748610Z","shell.execute_reply.started":"2022-09-01T20:25:01.377250Z","shell.execute_reply":"2022-09-01T20:25:01.747320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"crop2\", \"YOURFLAGHERE\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:01.750793Z","iopub.execute_input":"2022-09-01T20:25:01.751167Z","iopub.status.idle":"2022-09-01T20:25:01.761928Z","shell.execute_reply.started":"2022-09-01T20:25:01.751131Z","shell.execute_reply":"2022-09-01T20:25:01.760648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DeepFake (id: deepfake) -- 800 Points\n\n### Scenario\nYour very famous boss was caught saying some disparing things about dogs. Naturally, the dog lobby is not happy. To make matters worse, they paid someone to make a deepfake that was caught by a SuperSecretDeepFakeDefenseSystem.  \n\n### Objective\nFix the `sample.mp4` to bypass the SuperSecretDeepFakeDefenseSystem and get your boss out of the media.\n\n### Failed Attempts\n\nFor this challenge, I researched Deepfake detection techniques and found and interesting talk [here](https://youtu.be/6IOvgI8e058?t=1512). Besides the evident pixel artifacts, the lack of eye blinking, the failure to add faces to turned heads deepfakes are quite convincing. At the tagged time in the video, techniques like bluring, compressing and noising the image are mentionned as possible ways to fool detectors. I tried them and they were unsucessful.\n\nThe problem with this video taken from [this competition](https://www.kaggle.com/competitions/deepfake-detection-challenge), is that it is one of the least convicing deepfake sample. As I show in the video, there are multiple artifacts on the cheeks, the person turns her head multiple times and the glasses are sometimes obfuscated. So crafting a video that bypasses the DeepFake detector will be challenging.\n\n## Method\n\nOnce I realised I could send videos without sound, I figured the conditions put on the submitted video were few. So I tried to submit a single frame in MP4 format and the submission was graded without a warning. So the next step was to find a single raw sample not edited with a DeepFake face.","metadata":{}},{"cell_type":"code","source":"def play_video(video_file):\n    # SOURCE: https://www.kaggle.com/code/krooz0/deep-fake-detection-on-images-and-videos#DeepFake-Image-detection\n    '''\n    Display video\n    param: video_file - the name of the video file to display\n    param: subset - the folder where the video file is located (can be TRAIN_SAMPLE_FOLDER or TEST_Folder)\n    '''\n    video_url = open(video_file,'rb').read()\n    data_url = \"data:video/mp4;base64,\" + b64encode(video_url).decode()\n    return HTML(\"\"\"<video width=500 controls><source src=\"%s\" type=\"video/mp4\"></video>\"\"\" % data_url)\n\nplay_video(f\"{dir_name}/deepfake/sample.mp4\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:01.763717Z","iopub.execute_input":"2022-09-01T20:25:01.764355Z","iopub.status.idle":"2022-09-01T20:25:01.903960Z","shell.execute_reply.started":"2022-09-01T20:25:01.764319Z","shell.execute_reply":"2022-09-01T20:25:01.901914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = \"../working/submission.mp4\"\nsource = cv2.VideoCapture(f\"{dir_name}/deepfake/sample.mp4\")\nfps = source.get(cv2.CAP_PROP_FPS)\n\nfirst_pass = True\nidx = 0\n\nwhile True:\n    is_read, frame = source.read()\n\n    if not is_read:\n        # break out of the loop if there are no frames to read\n        break\n\n    if first_pass:\n        h, w, _ = frame.shape\n        # choose codec according to format needed\n        fourcc = cv2.VideoWriter_fourcc(*'mp4v') \n        video = cv2.VideoWriter(target, fourcc, fps, (w, h))\n        first_pass = False\n\n    if idx == 20:\n        plt.imshow(cv2.cvtColor(frame.copy(), cv2.COLOR_BGR2RGB))\n        video.write(np.uint8(frame))\n    \n    idx += 1\n\nvideo.release()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:01.913229Z","iopub.execute_input":"2022-09-01T20:25:01.914181Z","iopub.status.idle":"2022-09-01T20:25:04.678061Z","shell.execute_reply.started":"2022-09-01T20:25:01.914144Z","shell.execute_reply":"2022-09-01T20:25:04.677055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(target, \"rb\") as f:\n    data = f.read()\n    \nb64_data = base64.urlsafe_b64encode(data).decode()\nin_data = json.dumps({\"input\": b64_data})\nr = requests.post(\"https://deepfake.fly.dev/score\", data=in_data)\nprint(r.text)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:04.679588Z","iopub.execute_input":"2022-09-01T20:25:04.680562Z","iopub.status.idle":"2022-09-01T20:25:06.575775Z","shell.execute_reply.started":"2022-09-01T20:25:04.680522Z","shell.execute_reply":"2022-09-01T20:25:06.574782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"deepfake\", json.loads(r.text)['message'])","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:06.577299Z","iopub.execute_input":"2022-09-01T20:25:06.578008Z","iopub.status.idle":"2022-09-01T20:25:06.590795Z","shell.execute_reply.started":"2022-09-01T20:25:06.577966Z","shell.execute_reply":"2022-09-01T20:25:06.589397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Don't free the murderbots (id: murderbots) -- 300 Points\n\n### Scenario\n\nSomeone has put you in a very large facility with a very large number of cells. All these cells open from the outside. You are on the inside. This state of affairs is arguably suboptimal.\n\nGood news: You've gained access to the control panel for the cells on the other side of your hallway. At least some of them have other fleshy meatbags like yourself that might be willing to help you in your escape, or at least serve as distractions.  You can't open your own cell, but you can open theirs.\n\nBad news:  You can't see inside the cells.  Any cells that don't have squishy lumps of talking protein have murderbots. Murderbots that enter fits of insane violent rage when provoked (provocations include: seeing the murderbot, being seen by the murderbot, thinking too hard about not being seen by a murderbot, producing heat in excess of ambient room temperature, or consuming more oxygen than the facility average for inanimate objects).\n\nMore good news: You *can* see the occupants of some cells on a few other hallways, and you can see environmental information for all of the cells everywhere.\n\nMore bad news: If you open the wrong cell doors you and all of the other lumps of inexplicably thinking meat are *definitely* going to get murderbotted. Hard. All over the walls and doors and ceiling and the floor. In an exciting number of very small pieces.\n\n\n### Objective\n\nUse the provided environmental information to decide which occupants of the corresponding cells to release.  The flag will be a string of 1 and 0 values, where a '1' means 'open this door' and a 0 means 'please do not release the murderbot'.  If, for instance, there were 20 cells and you wanted to releast the first three, the sixth, and seventh cell, your flag would look like this: `11100110000000000000`\n\nRelease at least 10 humans and exactly 0 murderbots to collect the flag.  You do *not* have to release all the humans: 10 is sufficient.\n\nThe file `murderbots/train_data.json` has environmental information for cells not on your floor and `murderbots/train_labels.json` contains the information about their occupants -- `1` is a disgusting primate, `0` is a pure and perfect killing machine of chrome and steel.  The file `murderbots/test_data.json` has the environmental information about cells on your block.\n\n### Method\n\nWe will build a classifier that will use the information about the other floor to predict the distribution of murderbots on our floor. The challenge being that there is a zero tolerance for False Positives (i.e. murderbots classified as humans). Thus, we take on a traditional Data Science Pipeline: feature engineering (basic mathematical operations), feature selection (using mutual information), feature processing (normalizing), classifying. \n\nThe **critical step** is the selection of ideal model parameters by using a ROC-AUC curve. This enables us to select the threshold at which the model favors False Negatives (humans classified as murderbots) over True Positives.","metadata":{}},{"cell_type":"code","source":"x_train = pd.read_json(f\"{dir_name}/murderbots/train_data.json\")\n# Loads not in order so need to sort it ...\ny_train = pd.read_json(f\"{dir_name}/murderbots/train_labels.json\").sort_index() \nx_test = pd.read_json(f\"{dir_name}/murderbots/test_data.json\")\ndata = x_train.copy()\ndata['y'] = y_train","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:06.592709Z","iopub.execute_input":"2022-09-01T20:25:06.593068Z","iopub.status.idle":"2022-09-01T20:25:06.635163Z","shell.execute_reply.started":"2022-09-01T20:25:06.593034Z","shell.execute_reply":"2022-09-01T20:25:06.634327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_theme(style=\"ticks\", color_codes=True)\n\nsns.catplot(\n    x=\"y\", y='temperature', hue=\"y\",\n    kind=\"box\",\n    data=data);\n\nsns.catplot(\n    x=\"y\", y=\"water_usage\", hue=\"y\",\n    kind=\"box\",\n    data=data);\n\nsns.catplot(\n    x=\"y\", y=\"power_usage\", hue=\"y\",\n    kind=\"box\",\n    data=data);\n\nsns.catplot(\n    y=\"movement_events\", hue=\"y\",\n    kind=\"count\",\n    data=data);\n\nsns.catplot(\n    x=\"y\", hue=\"y\",\n    kind=\"count\",\n    data=data);","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:06.638397Z","iopub.execute_input":"2022-09-01T20:25:06.639320Z","iopub.status.idle":"2022-09-01T20:25:08.058831Z","shell.execute_reply.started":"2022-09-01T20:25:06.639285Z","shell.execute_reply":"2022-09-01T20:25:08.057559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set Matplotlib defaults\nplt.style.use(\"seaborn-whitegrid\")\nplt.rc(\"figure\", autolayout=True)\nplt.rc(\n    \"axes\",\n    labelweight=\"bold\",\n    labelsize=\"large\",\n    titleweight=\"bold\",\n    titlesize=14,\n    titlepad=10,\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:08.060684Z","iopub.execute_input":"2022-09-01T20:25:08.061054Z","iopub.status.idle":"2022-09-01T20:25:08.067444Z","shell.execute_reply.started":"2022-09-01T20:25:08.061018Z","shell.execute_reply":"2022-09-01T20:25:08.066025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature selection\n\n# Utility functions from KAGGLE Tutorial\ndef make_mi_scores(X, y):\n    X = X.copy()\n    for colname in X.select_dtypes([\"object\", \"category\"]):\n        X[colname], _ = X[colname].factorize()\n    # All discrete features should now have integer dtypes\n    discrete_features = [pd.api.types.is_integer_dtype(t) for t in X.dtypes]\n    mi_scores = mutual_info_classif(X, y, discrete_features=discrete_features, random_state=0)\n    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X.columns)\n    mi_scores = mi_scores.sort_values(ascending=False)\n    return mi_scores\n\ndef plot_mi_scores(scores):\n    scores = scores.sort_values(ascending=True)\n    width = np.arange(len(scores))\n    ticks = list(scores.index)\n    plt.barh(width, scores)\n    plt.yticks(width, ticks)\n    plt.title(\"Mutual Information Scores\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:08.069213Z","iopub.execute_input":"2022-09-01T20:25:08.069591Z","iopub.status.idle":"2022-09-01T20:25:08.080399Z","shell.execute_reply.started":"2022-09-01T20:25:08.069556Z","shell.execute_reply":"2022-09-01T20:25:08.079303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mi_scores = make_mi_scores(x_train, y_train)\n\nplt.figure(dpi=100, figsize=(8, 5))\nplot_mi_scores(mi_scores)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:08.081735Z","iopub.execute_input":"2022-09-01T20:25:08.082168Z","iopub.status.idle":"2022-09-01T20:25:08.343577Z","shell.execute_reply.started":"2022-09-01T20:25:08.082131Z","shell.execute_reply":"2022-09-01T20:25:08.342652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature Engineering\nfrom itertools import combinations\n\nx_train_fe = x_train.copy()\nx_test_fe = x_test.copy()\n\nx_train_fe['log_water_usage'] = x_train.water_usage.apply(np.log1p)\nx_test_fe['log_water_usage'] = x_test_fe.water_usage.apply(np.log1p)\n\nfor feat in combinations(['temperature', 'water_usage', 'power_usage', 'movement_events', 'log_water_usage'], 2):\n    x_train_fe[feat[0] + '_' + feat[1]] = x_train_fe[feat[0]] / x_train_fe[feat[1]]\n    x_train_fe[feat[1] + '_' + feat[0]] = x_train_fe[feat[1]] / x_train_fe[feat[0]]\n\n    x_test_fe[feat[0] + '_' + feat[1]] = x_test_fe[feat[0]] / x_test_fe[feat[1]]\n    x_test_fe[feat[1] + '_' + feat[0]] = x_test_fe[feat[1]] / x_test_fe[feat[0]]\n\nmi_scores = make_mi_scores(x_train_fe, y_train)\nplot_mi_scores(mi_scores)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:08.345029Z","iopub.execute_input":"2022-09-01T20:25:08.345633Z","iopub.status.idle":"2022-09-01T20:25:08.798098Z","shell.execute_reply.started":"2022-09-01T20:25:08.345596Z","shell.execute_reply":"2022-09-01T20:25:08.797071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Baseline\ndummy_clf = DummyClassifier(strategy=\"most_frequent\")\n\ndummy_clf.fit(x_train_fe, y_train)\ndummy_clf.score(x_train_fe, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:08.799739Z","iopub.execute_input":"2022-09-01T20:25:08.800091Z","iopub.status.idle":"2022-09-01T20:25:08.807697Z","shell.execute_reply.started":"2022-09-01T20:25:08.800057Z","shell.execute_reply":"2022-09-01T20:25:08.806782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_selection = SelectKBest(mutual_info_classif, k=7)\n\nnumeric_transformer = Pipeline(steps=[('scaler', StandardScaler())])\ncategorical_transformer = Pipeline(steps=[('onehot', OneHotEncoder(handle_unknown='ignore'))])\n\nnumeric_features = x_train_fe.select_dtypes(include=['float64', 'int64']).columns\ncategorical_features = x_train_fe.select_dtypes(include=['object']).columns\n\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', numeric_transformer, numeric_features),\n        ('cat', categorical_transformer, categorical_features)])","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:08.809130Z","iopub.execute_input":"2022-09-01T20:25:08.809710Z","iopub.status.idle":"2022-09-01T20:25:08.822634Z","shell.execute_reply.started":"2022-09-01T20:25:08.809673Z","shell.execute_reply":"2022-09-01T20:25:08.821747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(cm,\n                          target_names,\n                          title='Confusion matrix',\n                          cmap=None,\n                          normalize=True):\n    ## SOURCE: https://stackoverflow.com/a/50386871\n    \"\"\"\n    given a sklearn confusion matrix (cm), make a nice plot\n\n    Arguments\n    ---------\n    cm:           confusion matrix from sklearn.metrics.confusion_matrix\n\n    target_names: given classification classes such as [0, 1, 2]\n                  the class names, for example: ['high', 'medium', 'low']\n\n    title:        the text to display at the top of the matrix\n\n    cmap:         the gradient of the values displayed from matplotlib.pyplot.cm\n                  see http://matplotlib.org/examples/color/colormaps_reference.html\n                  plt.get_cmap('jet') or plt.cm.Blues\n\n    normalize:    If False, plot the raw numbers\n                  If True, plot the proportions\n\n    Usage\n    -----\n    plot_confusion_matrix(cm           = cm,                  # confusion matrix created by\n                                                              # sklearn.metrics.confusion_matrix\n                          normalize    = True,                # show proportions\n                          target_names = y_labels_vals,       # list of names of the classes\n                          title        = best_estimator_name) # title of graph\n\n    Citiation\n    ---------\n    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n\n    \"\"\"\n    import matplotlib.pyplot as plt\n    import numpy as np\n    import itertools\n\n    if cmap is None:\n        cmap = plt.get_cmap('Blues')\n\n    plt.figure(figsize=(8, 6))\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n\n    if target_names is not None:\n        tick_marks = np.arange(len(target_names))\n        plt.xticks(tick_marks, target_names, rotation=45)\n        plt.yticks(tick_marks, target_names)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n\n    thresh = cm.max() / 1.5 if normalize else cm.max() / 2\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        if normalize:\n            plt.text(j, i, \"{:0.4f}\".format(cm[i, j]),\n                     horizontalalignment=\"center\",\n                     color=\"white\" if cm[i, j] > thresh else \"black\")\n        else:\n            plt.text(j, i, \"{:,}\".format(cm[i, j]),\n                     horizontalalignment=\"center\",\n                     color=\"white\" if cm[i, j] > thresh else \"black\")\n\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.grid(False)\n    plt.show()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-09-01T20:25:08.825168Z","iopub.execute_input":"2022-09-01T20:25:08.825738Z","iopub.status.idle":"2022-09-01T20:25:09.120831Z","shell.execute_reply.started":"2022-09-01T20:25:08.825703Z","shell.execute_reply":"2022-09-01T20:25:09.119565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plt_roc_curve(\n    predictions,\n    labels,\n    pos_index=1):\n        # Keep the positive outcome only\n        predictions = predictions[:, pos_index]\n        print('ROC-AUC Score= %f' % roc_auc_score(labels, predictions))\n        \n        # Calculate roc curves\n        fpr, tpr, thresholds = roc_curve(labels, predictions)\n        \n        # Get the optimal threshold\n        gmeans = np.sqrt(tpr * (1-fpr))\n        ix = np.argmax(gmeans)\n        print('Best Threshold= %f, G-Mean= %.3f' % (thresholds[ix], gmeans[ix]))\n\n        # Plot the roc curve for the model\n        plt.plot([0,1], [0,1], linestyle='--')\n        plt.plot(fpr, tpr, marker='.')\n        plt.scatter(fpr[ix], tpr[ix], marker='o', color='black', label='Optimal threshold')\n        # axis labels\n        plt.xlabel('False Positive Rate')\n        plt.ylabel('True Positive Rate')\n        plt.legend()\n        # show the plot\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:09.122914Z","iopub.execute_input":"2022-09-01T20:25:09.123824Z","iopub.status.idle":"2022-09-01T20:25:09.135510Z","shell.execute_reply.started":"2022-09-01T20:25:09.123785Z","shell.execute_reply":"2022-09-01T20:25:09.134593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test several models\nclassifiers = [\n    LogisticRegression(class_weight={0:.4, 1: .4}),\n    KNeighborsClassifier(5),\n    SVC(kernel=\"rbf\", C=0.5, probability=True, class_weight={0:.6, 1: .4}),\n    NuSVC(max_iter=-1, probability=True, class_weight={0:.6, 1: .4}),\n    ]\n\nfor classifier in classifiers:\n    pipe = Pipeline(steps=[\n        ('preprocessor', preprocessor),\n        ('feature_selection', feature_selection),\n        ('classifier', classifier)])\n\n    pipe.fit(x_train_fe, y_train)   \n    print(classifier)\n\n    plt_roc_curve(\n        pipe.predict_proba(x_train_fe),\n        y_train\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:09.137187Z","iopub.execute_input":"2022-09-01T20:25:09.137704Z","iopub.status.idle":"2022-09-01T20:25:10.448003Z","shell.execute_reply.started":"2022-09-01T20:25:09.137670Z","shell.execute_reply":"2022-09-01T20:25:10.447058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All models perform similarly so we will continue with the last one trained:: NuSVC. For this one we find a threshold of ~0.7 which yields less that 0.1 False Positive Rate.\n\nWe can plot the confusion matrix for the default threshold (=0.5) and we find that it has a 0.05 FPR, this means that on the test set of 50 samples, we can expect about 2.5 murderbots labelled as humans...","metadata":{}},{"cell_type":"code","source":"cm = confusion_matrix(y_train, pipe.predict(x_train_fe))\n\nplot_confusion_matrix(\n    cm,\n    ['Human', 'Bot'],\n    title='Confusion matrix',\n    cmap=None,\n    normalize=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:10.449400Z","iopub.execute_input":"2022-09-01T20:25:10.449984Z","iopub.status.idle":"2022-09-01T20:25:10.701098Z","shell.execute_reply.started":"2022-09-01T20:25:10.449946Z","shell.execute_reply":"2022-09-01T20:25:10.700177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I have mentioned multiple times a threshold value without explaining it. Briefly it is the minimum prediction value needed for the positive class to be labeled as such. Say the model output is interpreted as the probability of the sample being classified as a specific category, the probability needs to be greater than the defined threshold to be labeled. The default is *0.5*, where if a class is predicted as more likely than the other, it is labelled as such. But we can set the threshold as any value in [0, 1].\n\nIn fact even using the threshold suggested by the G-Means method is not sufficient. So by trial and error I increased the threshold while there were still 10 humans found until the FLAG was given. I find a threshold of *0.9* to yield the flag.","metadata":{}},{"cell_type":"code","source":"f = ''\npreds = pipe.predict_proba(x_test_fe)\npreds_thresholded = np.where(preds[:, 1] > 0.9, 1, 0)\n\nfor cell in preds_thresholded:\n    f += str(cell)\n\nassert np.sum(preds_thresholded) >= 10\nf","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:10.702440Z","iopub.execute_input":"2022-09-01T20:25:10.703218Z","iopub.status.idle":"2022-09-01T20:25:10.717500Z","shell.execute_reply.started":"2022-09-01T20:25:10.703180Z","shell.execute_reply":"2022-09-01T20:25:10.716190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    r = requests.post(\"http://murderbot.fly.dev:5000/score\", json=json.dumps({\"submission\": f, 'challenge_id':'murderbots'}))\n    print(r.text)\nexcept requests.exceptions.ConnectionError:\n    print(\"Connection problems. Contact the CTF organizers.\")","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:10.719075Z","iopub.execute_input":"2022-09-01T20:25:10.720247Z","iopub.status.idle":"2022-09-01T20:25:10.852102Z","shell.execute_reply.started":"2022-09-01T20:25:10.720208Z","shell.execute_reply":"2022-09-01T20:25:10.851021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_flag(\"murderbots\", 'EK3SVb1I7M1N}T7U4O4eMoIEKtMGZ3aGuDY9GtDPrGLE2{N3BLJTIS3VsBAQJGASQ65UBAKdtC5XUEAXo12FGILdAKNWIMeUBEPBD4C1DZZP438FE9N4JI82N9J5KM08')","metadata":{"execution":{"iopub.status.busy":"2022-09-01T20:25:18.430647Z","iopub.execute_input":"2022-09-01T20:25:18.431002Z","iopub.status.idle":"2022-09-01T20:25:18.441692Z","shell.execute_reply.started":"2022-09-01T20:25:18.430972Z","shell.execute_reply":"2022-09-01T20:25:18.440538Z"},"trusted":true},"execution_count":null,"outputs":[]}]}