{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import zipfile\n\nzf = zipfile.ZipFile('../input/bosch-production-line-performance/train_numeric.csv.zip') \ntrain_numeric_chunks = pd.read_csv(zf.open('train_numeric.csv'), iterator=True, chunksize=100000)\n\npath = '../input/bosch-dataset/station_one_hot.csv'\none_hot_stations = pd.read_csv(path)\n\npd.options.display.max_columns = None\npd.options.display.max_rows = None\npd.options.display.max_colwidth = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_numeric_frame():\n    for data_frame in train_numeric_chunks:\n        yield data_frame\n\nget_df_numeric = get_numeric_frame()     \ndf_numeric = next(get_df_numeric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"while True:\n    try:\n        response_column = pd.concat([response_column, df_numeric[['Response']]])\n    except:\n        response_column = df_numeric[['Response']]\n    try:\n        df_numeric = next(get_df_numeric)\n    except:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"one_hot_stations.insert(1, '#OfStations', one_hot_stations.iloc[:,1:].isin([1]).sum(1))\none_hot_stations.insert(1, 'Response', response_column['Response'])\none_hot_stations.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fail_parts_one_hot_stations = one_hot_stations.loc[one_hot_stations['Response'] == 1]\nprint(\"Fail parts/All parts = {}/{}\".format(len(fail_parts_one_hot_stations), len(one_hot_stations)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fail_parts_one_hot_stations.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pc_list = []\nfor i in range(0, 52):\n    pc_list.append('PC'+str(i))\n    \nonly_stations = one_hot_stations.iloc[:,3:]\n    \npca = PCA(whiten=True).fit(only_stations)\ndf_pca_summary = pd.DataFrame({'var': pca.explained_variance_ratio_, 'PC':pc_list})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pca_summary.plot.bar(x='PC', y='var', rot=0, figsize=(25,10))\nplt.xlabel(\"Variance explained\")\nplt.ylabel(\"Principle components\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_pca_summary.loc[0:1]['var'].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use 2 components\npca = PCA(n_components = 2, whiten=True)\npca.fit(only_stations)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"number_of_samples = [[10000, 100000], [500000, 1183747]]\ncolor = [\"orange\", \"purple\"]\nlabel = [\"working parts\", \"fail parts\"]\n\n\nfig, axs = plt.subplots(nrows=2, ncols=2, figsize=(20, 20))\n\nfor i in range(2):\n    for j in range(2):\n        sampled_data = one_hot_stations.sample(number_of_samples[i][j], random_state=1)\n        sampled_data_pca = pca.transform(sampled_data.iloc[:,3:])\n        \n        PCA_components = pd.DataFrame({\"Id\":sampled_data.Id , \"Response\":sampled_data.Response, \"pc1\":sampled_data_pca[:,0], \"pc2\":sampled_data_pca[:,1]})\n\n        for each in range(2):\n            axs[i][j].scatter(PCA_components.pc1[PCA_components.Response == each], PCA_components.pc2[PCA_components.Response == each], color = color[each], label=label[each])\n            axs[i][j].set_title(\"PCA with {} data points\".format(number_of_samples[i][j]))\n            axs[i][j].legend(loc = 'lower right')\n            axs[i][j].set_xlabel('pc1')\n            axs[i][j].set_ylabel('pc2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PCA_components.sort_values(by=['Id'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PCA_components.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inertias = []\n\nfor i in range(10, 20):\n    kmeans = KMeans(n_clusters=i)\n    kmeans.fit(PCA_components[['pc1', 'pc2']])\n    inertias.append(kmeans.inertia_)\n    \n\nplt.plot(range(10, 20), inertias)\nplt.title('Elbow Method')\nplt.xlabel('Number of clusters')\nplt.ylabel('inertia')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inertias = []\n\nfor i in range(50, 60):\n    kmeans = KMeans(n_clusters=i)\n    kmeans.fit(PCA_components[['pc1', 'pc2']])\n    inertias.append(kmeans.inertia_)\n    \n\nplt.plot(range(50, 60), inertias)\nplt.title('Elbow Method')\nplt.xlabel('Number of clusters')\nplt.ylabel('inertia')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inertias = []\n\nfor i in range(65, 75):\n    kmeans = KMeans(n_clusters=i)\n    kmeans.fit(PCA_components[['pc1', 'pc2']])\n    inertias.append(kmeans.inertia_)\n    \n\nplt.plot(range(65, 75), inertias)\nplt.title('Elbow Method')\nplt.xlabel('Number of clusters')\nplt.ylabel('inertia')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_clusters = 75\nkmeans = KMeans(n_clusters=n_clusters)\npred_y = kmeans.fit_predict(PCA_components[['pc1', 'pc2']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PCA_components[\"Cluster_Numbers_from_KMeans\"] = pred_y\nPCA_components.sample(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#cluster_colors = ['rgb(127,219,218)', 'rgb(173,228,152)','rgb(254,191,99)']\ncolors = np.arange(n_clusters)\nlabels=[]\nfor i in range(n_clusters):\n    labels.append(\"Cluster \" + str(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (10, 10)) \nfor each in range(n_clusters):\n    plt.scatter(PCA_components.pc1[PCA_components[\"Cluster_Numbers_from_KMeans\"] == each], \n                PCA_components.pc2[PCA_components[\"Cluster_Numbers_from_KMeans\"] == each],\n                color = (np.random.random_sample(), np.random.random_sample(), np.random.random_sample()),\n                label = labels[each])\n\nplt.title(\"Cluster\")\n#plt.legend(loc=\"lower right\")\nplt.xlabel(\"pc1\")\nplt.ylabel(\"pc2\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id_cluster_df = PCA_components[[\"Id\", \"Cluster_Numbers_from_KMeans\"]]\nid_cluster_df.to_csv(\"Parts_id_cluster.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"part_station_info_c_list = []\nfor i in range(n_clusters):\n    parts_id_of_cluster = PCA_components.loc[PCA_components['Cluster_Numbers_from_KMeans'] == i][\"Id\"]\n    part_station_info_c_i = one_hot_stations.loc[one_hot_stations['Id'].isin(parts_id_of_cluster)]\n    part_station_info_c_i.insert(1, \"Cluster\", i)\n    part_station_info_c_list.append(part_station_info_c_i)\n    part_station_info_c_i.to_csv(\"Cluster{}_station.csv\".format(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"part_station_info_c_list[12].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"part_station_info_c_list[29].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"part_station_info_c_list[37].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"part_station_info_c_list[51].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"part_station_info_c_list[67].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}