{"cells":[{"metadata":{"_uuid":"abfd66d38775c6c89be19af1a48b6878ba100a2a"},"cell_type":"markdown","source":"This notebook is an attempt to explore the merchants.csv file.\n1.  Do basic pre-processing\n2. Reduce dimensionality using an auto-encoder.\n3. Cluster Merchants based on the reduced dimensions"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = (12, 12) ### Setting the size of the Plots\n\nfrom keras.layers import Input, Dense\nfrom keras.models import Model\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#### Lets load the file\nmerchant_df = pd.read_csv(\"../input/merchants.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0566ec6d2883f630190afd12ae9fabbc256e20ce"},"cell_type":"code","source":"#### Lets take a quick look at the basic info. some of the columns have missing values\nmerchant_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a084164621246f86fafa6277e5a85b9c639525d"},"cell_type":"code","source":"#### Next lets look at the numeric values and see if everything is ok. \nmerchant_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ca4b7a9de7ed8f624a618042c37fdc3c32c90d0"},"cell_type":"code","source":"#### We can see that 3 columns have the maximum value as inf, Lets see how many rows are affected\nprint(merchant_df[merchant_df['avg_purchases_lag3']==float('Inf')].merchant_id.count())\nprint(merchant_df[merchant_df['avg_purchases_lag6']==float('Inf')].merchant_id.count())\nprint(merchant_df[merchant_df['avg_purchases_lag12']==float('Inf')].merchant_id.count())\nmerchant_df[merchant_df['avg_purchases_lag3']==float('Inf')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b3760315b18ffc36cd694781df216ef52bcab47"},"cell_type":"code","source":"#### We can see that only 3 rows are affected. we can drop these rows\nmerchant_df = merchant_df.drop(merchant_df.index[merchant_df.avg_purchases_lag3 ==float('Inf')],0)\nmerchant_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9888b3814a68ecd7d96607e516fb2767a532bd4"},"cell_type":"code","source":"#### Lets look at handling some of the outliers in the sales lag / purchase lag fields\nmerchant_df.plot.scatter(x='avg_sales_lag3', y='avg_purchases_lag3')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71092638cd3a9b2d405cef49e9d2e74344ba7a4b"},"cell_type":"code","source":"### we can see only 16 records are getting affected. Lets drop these\n### We can repeat the scatter plot for avg_sales_lag6, avg_sales_lag12 as well\nprint(merchant_df[merchant_df['avg_sales_lag3']>20000].merchant_id.count())\nmerchant_df = merchant_df.drop(merchant_df.index[merchant_df.avg_sales_lag3>20000],0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c399372790bcfe7797be59a0f7256eda47a7a7e4"},"cell_type":"code","source":"merchant_df.plot.scatter(x='avg_sales_lag6', y='avg_purchases_lag6')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47d6acac3af8068a01eb695a8b3f59fea7fc4c7a"},"cell_type":"code","source":"### we can see only 16 records are getting affected. Lets drop these\n### We can repeat the scatter plot for avg_sales_lag6, avg_sales_lag12 as well\nprint(merchant_df[merchant_df['avg_sales_lag6']>20000].merchant_id.count())\nmerchant_df = merchant_df.drop(merchant_df.index[merchant_df.avg_sales_lag6>20000],0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c68328e6504da838302555a729e42fa870d3a63c"},"cell_type":"code","source":"merchant_df.plot.scatter(x='avg_sales_lag12', y='avg_purchases_lag12')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"258ce462585c72d1df869871f48e39502f97751a"},"cell_type":"code","source":"### we can see only 16 records are getting affected. Lets drop these\n### We can repeat the scatter plot for avg_sales_lag6, avg_sales_lag12 as well\nprint(merchant_df[merchant_df['avg_sales_lag12']>20000].merchant_id.count())\nmerchant_df = merchant_df.drop(merchant_df.index[merchant_df.avg_sales_lag12>20000],0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c44388a3e3dc5833e3e6397fca4d97b3c472418f"},"cell_type":"code","source":"merchant_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e65e4792ef36240ac80b12d660c9199e039ddfe3"},"cell_type":"code","source":"##### Next Lets handle the 4 categorical values\n### category_1, category_4  have Y or N we can replace this with 0 / 1\n### most_recent_sales_range, most_recent_purchases_range are kind of ranking of some sort with A>B>C>D>E, we can replace these with 5>4>3>2>1\nclean_up_categoricals = {'category_1':{'Y':1, 'N':0},\n                         'category_4' :{'Y':1, 'N':0},\n                        'most_recent_sales_range' : {'A':5,'B':4,'C':3,'D':2,'E':1},\n                        'most_recent_purchases_range' : {'A':5,'B':4,'C':3,'D':2,'E':1}}\nmerchant_df.replace(clean_up_categoricals, inplace=True)\nmerchant_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfaff40822fb654c41a3eb279569a9ab728994ac"},"cell_type":"code","source":"##### Finally lets replace the null values with the respective means of the columns\nmerchant_df = merchant_df.fillna(merchant_df.mean())\nmerchant_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c343827c68df148378622fcd9329c29d30dcaa35"},"cell_type":"code","source":"#### Next we can extract our X component out of this - All fields excluding the merchant_id\nX = merchant_df\nX = X.drop('merchant_id',1)\nX.head(2)\nX.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad1d260786e1df47ae943fb329ead4dc5a5872cb"},"cell_type":"code","source":"#### Lets use the standard scaler to scale all our columns\nfrom sklearn.preprocessing import StandardScaler\nsc_X = StandardScaler()\nX = sc_X.fit_transform(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40a8768c0aacccd22163b8f49b64a3d783dc64ea"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test = train_test_split(X, test_size = 0.2, random_state = 0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b73c3c9323a29d8d276504cbc50408b83fe4e25"},"cell_type":"code","source":"# this is the size of our encoded representations\nencoding_dim = 6  # we need our 21 columns to be encoded as 6\n# this is our input placeholder\ninput_shape = Input(shape=(21,))\n# \"encoded\" is the encoded representation of the input\nencoded = Dense(encoding_dim, activation='relu')(input_shape)\n# \"decoded\" is the lossy reconstruction of the input\ndecoded = Dense(21, activation='sigmoid')(encoded)\n# this model maps an input to its reconstruction\nautoencoder = Model(input_shape, decoded)\n# this model maps an input to its encoded representation\nencoder = Model(input_shape, encoded)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"489565228a46a7df9068e92f6fdf0ef64b53b021"},"cell_type":"code","source":"# create a placeholder for an encoded (32-dimensional) input\nencoded_input = Input(shape=(encoding_dim,))\n# retrieve the last layer of the autoencoder model\ndecoder_layer = autoencoder.layers[-1]\n# create the decoder model\ndecoder = Model(encoded_input, decoder_layer(encoded_input))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2357c9a1cbc3233751741e93cb319de4336aab1"},"cell_type":"code","source":"#### Compile the autoencoder\nautoencoder.compile(optimizer='adadelta', loss='MSE')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0dd5fd6d6b3941eea405f2e4bd2681993e9aac57"},"cell_type":"code","source":"#### Train the autoencoder\nautoencoder.fit(X_train, X_train,\n                epochs=100,\n                batch_size=256,\n                shuffle=True,\n                validation_data=(X_test, X_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6159a3fdc49603f4773cbcc1248e75bae26a638"},"cell_type":"code","source":"#### Get our encoded data\nencoded_data = encoder.predict(X)\nencoded_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c02d30dfa35d7054fc5b8cb5772d6f0a975cbd30"},"cell_type":"code","source":"#### Lets find the optimal number of clusters using the elbow method by plotting wcss against the number of clusters\nfrom sklearn.cluster import KMeans\nwcss = []\n\nfor i in range (1,11):\n    kmeans = KMeans(n_clusters = i, init = \"k-means++\", max_iter = 300,\n                    n_init = 10, random_state = 0)\n    kmeans.fit(encoded_data)\n    wcss.append(kmeans.inertia_)\n\nplt.plot(range(1,11),wcss)\nplt.title(\"The Elbow Method\")\nplt.xlabel(\"Number of Clusters\")\nplt.ylabel(\"WCSS\")\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a312541c953643ce7413b5d6750800a7d51d6fb"},"cell_type":"code","source":"##### 4 seems to be a good number of clusters\n\nkmeans = KMeans(n_clusters = 4, init = \"k-means++\", max_iter = 300,\n                    n_init = 10, random_state = 0)\nykmeans = kmeans.fit_predict(encoded_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc0326e911d650d37276265e843f1421375942fb"},"cell_type":"code","source":"plt.scatter(encoded_data[ykmeans==0,0], encoded_data[ykmeans==0,1], s = 10, c='red', label = \"1\")\nplt.scatter(encoded_data[ykmeans==1,0], encoded_data[ykmeans==1,1], s = 10, c='blue', label = \"2\")\nplt.scatter(encoded_data[ykmeans==2,0], encoded_data[ykmeans==2,1], s = 10, c='green', label = \"3\")\nplt.scatter(encoded_data[ykmeans==3,0], encoded_data[ykmeans==3,1], s = 10, c='cyan', label = \"4\")\n\nplt.scatter(kmeans.cluster_centers_[:,0],kmeans.cluster_centers_[:,1], s= 100, c=\"yellow\", label = \"Centroids\")\nplt.title(\"Clusters of Clients\")\nplt.xlabel(\"X\")\nplt.ylabel(\"Y\")\nplt.legend()\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c35d1e4f1a11b0a3661b7024aa7b2c325c74a2b"},"cell_type":"code","source":"##### Finally Lets Create a new dataframe that will have just 2 columns - the merchant_id and the corresponding cluster id\nmerchant_id = merchant_df['merchant_id']\nprint(merchant_id.shape)\nprint(ykmeans.shape)\n\nnew_df = pd.DataFrame()\nnew_df['merchant_id'] = merchant_df['merchant_id']\nnew_df['cluster_id'] = ykmeans\nprint(new_df.info())\nnew_df.to_csv('merchant_id_clusters.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}