{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"L3_station = (30, 53)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/bosch-stations-one-hot-enc-train-test/stations_one_hot_train.csv'\none_hot_stations = pd.read_csv(path)\n\nL3_one_hot = one_hot_stations.iloc[:,L3_station[0]:L3_station[1]]\n\npd.options.display.max_columns = None\npd.options.display.max_rows = None\npd.options.display.max_colwidth = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Drop rows with all 0 for each station\nL3_one_hot = L3_one_hot.loc[~(L3_one_hot==0).all(axis=1)]\n\nprint(\"Parts in L3:{}\".format(len(L3_one_hot)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot.insert(0, \"Id\",one_hot_stations[\"Id\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom matplotlib import pyplot as plt\nfrom sklearn.cluster import DBSCAN","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DBSCAN for L3","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"column_names = L3_one_hot.columns[1:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot = L3_one_hot.sample(len(L3_one_hot))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dbscan = DBSCAN()\nnumber_of_splits = 20","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from random import randint\ncluster_number_list = []\n\nfor i in range(5):\n    split_number = randint(1, number_of_splits)\n    preds_split = dbscan.fit_predict(L3_one_hot[int((split_number-1)*len(L3_one_hot)/number_of_splits):int(split_number*len(L3_one_hot)/number_of_splits)][column_names])\n    cluster_number_list.append(len(np.unique(preds_split)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_cluster_for_l3 = int(np.max(cluster_number_list))\nprint(\"MAX NUMBER OF CLUSTERS: {}\\n\".format(max_cluster_for_l3))\nprint(cluster_number_list)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# KMEANS for L3","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot.sort_values(by=['Id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_clusters = max_cluster_for_l3\nkmeans = KMeans(n_clusters=n_clusters)\npred = kmeans.fit_predict(L3_one_hot[column_names])\n\npred += 1\nprint(kmeans.inertia_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot.insert(1, \"ClusterL3\", pred)\nL3_one_hot.sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ids_clusters = pd.DataFrame({\"Id\": one_hot_stations['Id'], \"ClusterL3\": 0})\nids_clusters.loc[L3_one_hot.index, ['ClusterL3']] = L3_one_hot['ClusterL3']\nids_clusters.to_csv(\"Cluster_L3_train.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot[L3_one_hot[\"ClusterL3\"] == randint(1, n_clusters)].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot[L3_one_hot[\"ClusterL3\"] == randint(1, n_clusters)].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot[L3_one_hot[\"ClusterL3\"] == randint(1, n_clusters)].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot[L3_one_hot[\"ClusterL3\"] == randint(1, n_clusters)].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# KMEANS ON TEST DATA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"path_test = '../input/bosch-stations-one-hot-enc-train-test/stations_one_hot_test.csv'\none_hot_stations_test = pd.read_csv(path_test)\n\nL3_one_hot_test = one_hot_stations_test.iloc[:,L3_station[0]:L3_station[1]]\n\n# Drop rows with all 0 for each station\nL3_one_hot_test = L3_one_hot_test.loc[~(L3_one_hot_test==0).all(axis=1)]\n\nprint(\"Parts in L3_test:{}\".format(len(L3_one_hot_test)))\n\nL3_one_hot_test.insert(0, \"Id\",one_hot_stations_test[\"Id\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test = kmeans.predict(L3_one_hot_test[column_names])\npred_test += 1\n\nL3_one_hot_test.insert(1, \"ClusterL3\", pred_test)\n\nids_clusters_test = pd.DataFrame({\"Id\": one_hot_stations_test['Id'], \"ClusterL3\": 0})\nids_clusters_test.loc[L3_one_hot_test.index, ['ClusterL3']] = L3_one_hot_test['ClusterL3']\nids_clusters_test.to_csv(\"Cluster_L3_test.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot_test[L3_one_hot_test[\"ClusterL3\"] == 1].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot[L3_one_hot[\"ClusterL3\"] == 1].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot_test[L3_one_hot_test[\"ClusterL3\"] == n_clusters].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"L3_one_hot[L3_one_hot[\"ClusterL3\"] == n_clusters].head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}