{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pathlib import Path\nimport os.path\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport seaborn as sns\n\n#Ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-12T22:17:40.917855Z","iopub.execute_input":"2022-05-12T22:17:40.918785Z","iopub.status.idle":"2022-05-12T22:18:48.269212Z","shell.execute_reply.started":"2022-05-12T22:17:40.918657Z","shell.execute_reply":"2022-05-12T22:18:48.265128Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Relationship between comment cells and code cells
\n\nThe Kaggle Team have assembled a dataset of approximately 160,000 public Python notebooks from Kaggle and have teamed up with X, the moonshot factory to design a competition that challenges participants to use this dataset of published notebooks to build creative techniques aimed at better understanding the relationship between comment cells and code cells.\n\nhttps://www.kaggle.com/competitions/AI4Code/overview/description\n\nHowever, a comment cell here is in fact a Markdown cell. Not just a comment.","metadata":{}},{"cell_type":"code","source":"import requests\nfrom bs4 import BeautifulSoup\nimport lxml\nimport os\nimport urllib\nimport sys\nimport cv2\nimport csv\nimport multiprocessing","metadata":{"execution":{"iopub.status.busy":"2022-05-12T22:19:11.753844Z","iopub.execute_input":"2022-05-12T22:19:11.754170Z","iopub.status.idle":"2022-05-12T22:19:12.267869Z","shell.execute_reply.started":"2022-05-12T22:19:11.754139Z","shell.execute_reply":"2022-05-12T22:19:12.267061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"What is a monotonic relationship?
\n\nBy Juhi Ramzai\n\n\"A monotonic relationship is a relationship that does one of the following:\"\n\n(1) as the value of one variable increases, so does the value of the other variable, OR,\"\n\n(2) as the value of one variable increases, the other variable value decreases.\"\n\n\"BUT, not exactly at a constant rate whereas in a linear relationship the rate of increase/decrease is constant.\"\n\nhttps://towardsdatascience.com/clearly-explained-pearson-v-s-spearman-correlation-coefficient-ada2f473b8#:~:text=What%20is%20a%20monotonic%20relationship,the%20other%20variable%20value%20decreases.","metadata":{}},{"cell_type":"markdown","source":"https://medium.com/analytics-vidhya/application-of-monotonic-constraints-in-machine-learning-models-334564bea616","metadata":{}},{"cell_type":"markdown","source":"\"Application of Monotonic Constraints in Machine Learning Models\"\n\nAuthor: Ajay Tiwari\n\n\"Monotonically Increasing Function — A function is called monotonically increasing if, for all x and y such that x≤y one has f(x)≤ f(y), so f preserves the order. This function does not exclusively have to increase, it simply must not decrease.\"\n\n\"Monotonically Decreasing Function — A function is called monotonically decreasing if, whenever x≤y, then f(x)≥f(y), so it reverses the order. This function does not exclusively have to decrease, it simply must not increase.\"\n\nhttps://medium.com/analytics-vidhya/application-of-monotonic-constraints-in-machine-learning-models-334564bea616","metadata":{}},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\nimport json, codecs\n\nwith codecs.open(\"../input/AI4Code/train/836790969cdeb2.json\", 'r',\n encoding='utf-8', errors='ignore') as f:\n train = json.load(f)\n \nwith codecs.open(\"../input/AI4Code/test/0009d135ece78d.json\", 'r',\n encoding='utf-8', errors='ignore') as f:\n test = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:06:27.429616Z","iopub.execute_input":"2022-05-12T23:06:27.432429Z","iopub.status.idle":"2022-05-12T23:06:27.455121Z","shell.execute_reply.started":"2022-05-12T23:06:27.432268Z","shell.execute_reply":"2022-05-12T23:06:27.453565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\ndisplay(train.keys())","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:06:35.273009Z","iopub.execute_input":"2022-05-12T23:06:35.274120Z","iopub.status.idle":"2022-05-12T23:06:35.284799Z","shell.execute_reply.started":"2022-05-12T23:06:35.274069Z","shell.execute_reply":"2022-05-12T23:06:35.283830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\ndisplay(test.keys())","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:06:42.370827Z","iopub.execute_input":"2022-05-12T23:06:42.371510Z","iopub.status.idle":"2022-05-12T23:06:42.378294Z","shell.execute_reply.started":"2022-05-12T23:06:42.371460Z","shell.execute_reply":"2022-05-12T23:06:42.377427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#To avoid Error \"If using all scalar values, you must pass an index\".\n\nThough it returned only one index and we wanted all.","metadata":{}},{"cell_type":"code","source":"#Code by Ventakumar R https://www.kaggle.com/venkatkumar001/hfp-2-eda-tensorflow/notebook\n\nmono = pd.DataFrame(train['cell_type'], index=[5])\n\ndisplay(mono)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:07:24.764775Z","iopub.execute_input":"2022-05-12T23:07:24.765581Z","iopub.status.idle":"2022-05-12T23:07:24.806327Z","shell.execute_reply.started":"2022-05-12T23:07:24.765511Z","shell.execute_reply":"2022-05-12T23:07:24.805331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#It's another trendy Covid-19 Notebook (confirmed/death/recovered).","metadata":{}},{"cell_type":"code","source":"mono1 = pd.DataFrame(train['source'], index=[5])\n\ndisplay(mono1)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:07:58.180712Z","iopub.execute_input":"2022-05-12T23:07:58.181269Z","iopub.status.idle":"2022-05-12T23:07:58.198561Z","shell.execute_reply.started":"2022-05-12T23:07:58.181227Z","shell.execute_reply":"2022-05-12T23:07:58.196771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Changing index doesn't seem to have changed anything","metadata":{}},{"cell_type":"code","source":"mono2 = pd.DataFrame(train['source'], index=[10])\n\ndisplay(mono2)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:09:03.418384Z","iopub.execute_input":"2022-05-12T23:09:03.419117Z","iopub.status.idle":"2022-05-12T23:09:03.434819Z","shell.execute_reply.started":"2022-05-12T23:09:03.419058Z","shell.execute_reply":"2022-05-12T23:09:03.434015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Checking another json file","metadata":{}},{"cell_type":"code","source":"with codecs.open(\"../input/AI4Code/train/5e0bc77bfaee72.json\", 'r',\n encoding='utf-8', errors='ignore') as f:\n train2 = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:09:40.772591Z","iopub.execute_input":"2022-05-12T23:09:40.773371Z","iopub.status.idle":"2022-05-12T23:09:40.782631Z","shell.execute_reply.started":"2022-05-12T23:09:40.773313Z","shell.execute_reply":"2022-05-12T23:09:40.781852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Some Global Terrorism Database","metadata":{}},{"cell_type":"code","source":"tonic = pd.DataFrame(train2['source'], index=[8])\n\ndisplay(tonic)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:10:13.297821Z","iopub.execute_input":"2022-05-12T23:10:13.298495Z","iopub.status.idle":"2022-05-12T23:10:13.331045Z","shell.execute_reply.started":"2022-05-12T23:10:13.298446Z","shell.execute_reply":"2022-05-12T23:10:13.329822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#First (only) row, 65th column \n\ntonic.iloc[0,64]","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:10:37.480300Z","iopub.execute_input":"2022-05-12T23:10:37.481114Z","iopub.status.idle":"2022-05-12T23:10:37.490137Z","shell.execute_reply.started":"2022-05-12T23:10:37.481061Z","shell.execute_reply":"2022-05-12T23:10:37.489065Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/AI4Code/train_orders.csv\", low_memory=False)\nprint(df.shape)\ndf.head().style.set_properties(**{'background-color':'#FFE4C4',\n 'color': 'black'})","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:13:58.641389Z","iopub.execute_input":"2022-05-12T23:13:58.642145Z","iopub.status.idle":"2022-05-12T23:14:00.390376Z","shell.execute_reply.started":"2022-05-12T23:13:58.642093Z","shell.execute_reply":"2022-05-12T23:14:00.389495Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anc = pd.read_csv(\"/kaggle/input/AI4Code/train_ancestors.csv\", low_memory=False)\nprint(anc.shape)\nanc.head().style.set_properties(**{'background-color':'#FFE4C4',\n 'color': 'black'})","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:17:55.558839Z","iopub.execute_input":"2022-05-12T23:17:55.559237Z","iopub.status.idle":"2022-05-12T23:17:55.789370Z","shell.execute_reply.started":"2022-05-12T23:17:55.559199Z","shell.execute_reply":"2022-05-12T23:17:55.788255Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n#fill in mean for floats\nfor c in df.columns:\n if df[c].dtype=='float16' or df[c].dtype=='float32' or df[c].dtype=='float64':\n df[c].fillna(df[c].mean())\n\n#fill in -999 for categoricals\ndf = df.fillna(-999)\n# Label Encoding\nfor f in df.columns:\n if df[f].dtype=='object': \n lbl = LabelEncoder()\n lbl.fit(list(df[f].values))\n df[f] = lbl.transform(list(df[f].values))\n \nprint('Labelling done.')","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:23:18.081822Z","iopub.execute_input":"2022-05-12T23:23:18.082622Z","iopub.status.idle":"2022-05-12T23:23:58.516825Z","shell.execute_reply.started":"2022-05-12T23:23:18.082547Z","shell.execute_reply":"2022-05-12T23:23:58.515138Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib\nimport statsmodels.tsa.api as tsa\n\n%matplotlib inline\n\nmatplotlib.style.use(\"Solarize_Light2\")\n\nfrom pylab import rcParams\nimport statsmodels.api as sm","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:25:36.056864Z","iopub.execute_input":"2022-05-12T23:25:36.057929Z","iopub.status.idle":"2022-05-12T23:25:36.932985Z","shell.execute_reply.started":"2022-05-12T23:25:36.057865Z","shell.execute_reply":"2022-05-12T23:25:36.931844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = df.loc[~df.id.isnull()][['id', 'cell_order']]\n\ny_log=np.log1p(y)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:24:11.788462Z","iopub.execute_input":"2022-05-12T23:24:11.788853Z","iopub.status.idle":"2022-05-12T23:24:11.822028Z","shell.execute_reply.started":"2022-05-12T23:24:11.788815Z","shell.execute_reply":"2022-05-12T23:24:11.821276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Oniel Gracious https://www.kaggle.com/onielg/simplecatboostwithclassifierchains/notebook\n\n#histogram and normal probability plot\nfrom scipy import stats\n\nfrom scipy.stats import norm\ntarget=['cell_order','id']\nfor t in target:\n fig = plt.figure()\n sns.distplot(y_log[t], fit=norm);\n res = stats.probplot(y_log[t], plot=plt)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:25:45.331901Z","iopub.execute_input":"2022-05-12T23:25:45.332492Z","iopub.status.idle":"2022-05-12T23:25:48.378673Z","shell.execute_reply.started":"2022-05-12T23:25:45.332453Z","shell.execute_reply":"2022-05-12T23:25:48.377672Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Pearson correlation. That competition applies Kendall Tau. ","metadata":{}},{"cell_type":"code","source":"corr = df.corr(method='pearson')\nsns.heatmap(corr);","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:39:00.619060Z","iopub.execute_input":"2022-05-12T23:39:00.619403Z","iopub.status.idle":"2022-05-12T23:39:01.612802Z","shell.execute_reply.started":"2022-05-12T23:39:00.619366Z","shell.execute_reply":"2022-05-12T23:39:01.612062Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Code by Amiya Ranjan Rout https://www.geeksforgeeks.org/python-kendall-rank-correlation-coefficient/\n\n# Import required libraries\nfrom scipy.stats import kendalltau\n \n# Taking values from the above example in Lists\nX = 'id'\nY = 'cell_order'\n \n# Calculating Kendall Rank correlation\ncorr, _ = kendalltau(X, Y)\nprint('Kendall Rank correlation: %.5f' % corr)\n \n# This code is contributed by Amiya Rout","metadata":{"execution":{"iopub.status.busy":"2022-05-12T23:42:28.608830Z","iopub.execute_input":"2022-05-12T23:42:28.609277Z","iopub.status.idle":"2022-05-12T23:42:28.618894Z","shell.execute_reply.started":"2022-05-12T23:42:28.609185Z","shell.execute_reply":"2022-05-12T23:42:28.617641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#At least, I tried to apply The Kendall Tau. By the way, the palette isn't beige is Bisque! ","metadata":{}},{"cell_type":"markdown","source":"softwaretestinghelp.com","metadata":{}}]}