{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\"\"\"\nIn this notebook I will see what I can do - EDA-wise without the additional data.\nI do that because it is not available for us in the test,\nmight be we an regenaerate them - but for now, the basic data is what I want to focus on.\n\"\"\"\n# loading the data\ntest = pd.read_csv(\"../input/test.csv\")\ntrain = pd.read_csv(\"../input/train.csv\")\nstructures = pd.read_csv(\"../input/structures.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# looking for Nan's in the data\n\nprint(train.isna().sum().sum())\nprint(structures.isna().sum().sum())\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"it seems like there are no nan's, but there is one more way to check this but isna,\nand that is histograms and peaks."},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train[\"id\"].nunique())\nprint(train[\"molecule_name\"].nunique())\nprint(train[\"id\"].nunique() - train.shape[0])\n\n# question, are there molecules that we compute only some of their couplings ?\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nFig, axarr= plt.subplots(2, 2, figsize=(12,8)) # define a grid with 2 rows, 1 columns - two             \n                                                                    #plots all in all insinde axarr\n\nPlt1 = train[\"atom_index_0\"].plot.hist(ax = axarr[0][0])\n\nPlt2 = train[\"atom_index_1\"].plot.hist(ax = axarr[0][1])\n\nPlt3 = train[\"type\"].value_counts().plot.hist(ax = axarr[1][0])\n\nPlt3 = train[\"scalar_coupling_constant\"].plot.hist(ax = axarr[1][1], bins =50)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nnow for the structures\n\"\"\"\nstructures.columns\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(structures[\"molecule_name\"].nunique())\nprint(structures[\"atom_index\"].nunique())\nprint(structures[\"atom\"].nunique())\nprint(\"x y z :\")\nprint(structures[\"x\"].nunique())\nprint(structures[\"y\"].nunique())\nprint(structures[\"z\"].nunique())\n\n\n# question : where did all these extra molecules came from? \n# answere: the structures contains molecules from both the train and the test.\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nFig, axarr= plt.subplots(2, 2, figsize=(12,8)) # define a grid with 2 rows, 1 columns - two             \n                                                                    #plots all in all insinde axarr\n\nPlt1 = structures[\"molecule_name\"].value_counts().plot.hist(ax = axarr[0][0], bins=50)\n\nPlt2 = structures[\"atom_index\"].plot.hist(ax = axarr[0][1])\n\nPlt3 = structures[\"atom\"].value_counts().plot.hist(ax = axarr[1][1])\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"OK, so most of the molecules has between 15 and 22 atom pairs in them, no appearent Nan."},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nlooking for outliers.\nsoo..\nwe are dealing with:\nmolecule_name', 'atom_index', 'atom', 'x', 'y', 'z'  - Structres.\n'id', 'molecule_name', 'atom_index_0', 'atom_index_1', 'type','scalar_coupling_constant'\n\"\"\"\n\n# lets join them up\nsample_data = train.sample(500)\nall_data = pd.merge(sample_data, structures, how = \"left\", on = \"molecule_name\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# now we have to think - what can be an outlier here ? \ntrain[\"scalar_coupling_constant\"].plot.hist(bins = 35)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nthere is a huge peack around the 0 of this constant.\nbut I do not think these are na values or outliers.\n\nlets now check if the train and test appeare to be from the same distrebution or not.\n\"\"\"\n\n# train: id, molecule_name, atom_index_0, atom_index_1, type, scalar_coupling_constant\n# test : id, molecule_name, atom_index_0, atom_index_1, type\n\n# I need to split structures to structures_train and structures_test\nprint(test.columns)\n\nstructures_train = structures.loc[structures[\"molecule_name\"].isin(train[\"molecule_name\"])]\nstructures_test = structures.loc[structures[\"molecule_name\"].isin(test['molecule_name'])]\n\nprint(structures_train.shape)\nprint(structures_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nso !!\nlets see what can I check..\n# train: id, molecule_name, atom_index_0, atom_index_1, type, scalar_coupling_constant\n# test : id, molecule_name, atom_index_0, atom_index_1, type\n# structures_train: 'molecule_name', 'atom_index', 'atom', 'x', 'y', 'z'\n# structures_test: 'molecule_name', 'atom_index', 'atom', 'x', 'y', 'z'\n\"\"\"\n\n\"\"\"\n1. type - is there roughly the same type distrebution in both ?\n2. atom - are there roughly the same atom distrebution in both ?\n3. x ?\n4. y ?\n5. z ?\n\"\"\"\n\n#1 type - is there roughly the same type distrebution in both ?\n\n\nFig, axarr= plt.subplots(1, 2, figsize=(12,8)) \n\nPlt1 = train[\"type\"].value_counts().plot.hist(ax = axarr[0])\n\nPlt2 = test[\"type\"].value_counts().plot.hist(ax = axarr[1])\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#2. atom - are there roughly the same atom distrebution in both ?\n\nFig, axarr= plt.subplots(1, 2, figsize=(12,8)) \n\nPlt1 = structures_train[\"atom\"].value_counts().plot.hist(ax = axarr[0])\n\nPlt2 = structures_test[\"atom\"].value_counts().plot.hist(ax = axarr[1])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# so .. yes, they are both from the same distrebution :O","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#next up ! relation plots ! lets get some features :)\n\n# train: id, molecule_name, atom_index_0, atom_index_1, type, scalar_coupling_constant\n# test : id, molecule_name, atom_index_0, atom_index_1, type\n# structures_train: 'molecule_name', 'atom_index', 'atom', 'x', 'y', 'z'\n# structures_test: 'molecule_name', 'atom_index', 'atom', 'x', 'y', 'z'\n\nbyType = train.groupby(\"type\")[\"scalar_coupling_constant\"].agg(\"median\")\nprint(byType)\n\n# note ! a huge differnce between types, if we have ourselves a tree-based model, \n# that will be fine - but any other kind of model must be seperated by the type or \n# manipulated in a diefferent way..\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"byAtom = all_data.groupby(\"atom\")[\"scalar_coupling_constant\"].agg(\"mean\")\nprint(byAtom)\n\"\"\"\nthis is wierd, the F atom is very dieffrent from all others.\n\"\"\"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}