{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Load Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.dates as md\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import host_subplot\nimport mpl_toolkits.axisartist as AA\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans\nfrom sklearn.covariance import EllipticEnvelope\nfrom sklearn.ensemble import IsolationForest\nfrom sklearn.svm import OneClassSVM\nfrom mpl_toolkits.mplot3d import Axes3D\n# from pyemma import msm\n%matplotlib inline\n\n# plt.style.use(\"fivethirtyeight\")\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load and filter Data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"expedia = pd.read_csv('/kaggle/input/expedia-personalized-sort/data/train.csv')\ndf = expedia.loc[expedia['prop_id'] == 104517]\ndf = df.loc[df['srch_room_count'] == 1]\ndf = df.loc[df['visitor_location_country_id'] == 219]\ndf = df[['date_time', 'price_usd', 'srch_booking_window', 'srch_saturday_night_bool']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check the Statistics of Anomaly Variable"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['price_usd'].describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**We have detect extream anomaly in `max : 5584.000000` value **\n\n> **If an individual data instance can be considered as anomalous with respect to the rest of the data, we call it Point Anomalies (e.g. purchase with large transaction value).**"},{"metadata":{"trusted":true},"cell_type":"code","source":"expedia.loc[(expedia['price_usd'] == 5584) & (expedia['visitor_location_country_id'] == 219)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **After Seeing the result we may see that either mistake or user is looking for Luxurious suits**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.loc[df['price_usd'] < 5584]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> ***We have to keep in mind that we don't have explaination about the room type like standard and Luxurious.***"},{"metadata":{},"cell_type":"markdown","source":"## Time series Data Visulization"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.plot(x='date_time', y = 'price_usd', figsize = (20,5))\nplt.xlabel('Date time')\nplt.ylabel('Price in USD')\nplt.title('Time Series of room price by date time of search')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Price Analysis count of Saturday night or Non-Saturday Night**"},{"metadata":{"trusted":true},"cell_type":"code","source":"a = df.loc[df['srch_saturday_night_bool'] == 0, 'price_usd']\nb = df.loc[df['srch_saturday_night_bool'] == 1, 'price_usd']\nplt.figure(figsize=(20, 6))\nplt.hist(a, bins = 50, alpha=0.5, label='Search Non-Sat Night')\nplt.hist(b, bins = 50, alpha=0.5, label='Search Sat Night')\nplt.legend(loc='upper right')\nplt.xlabel('Price\\n Price is more stable and lower when searching Non-Saturday night and price goes up when searching Saturday night', fontsize = 18)\nplt.ylabel('Count', fontsize = 18)\nplt.title(\"Price Comperision between Non-Saturday Night vs Saturday Night\")\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Different Model Approach\n\n### 1. K-means Algorithm\n* Similar cluster of data points we can find and the instance which is not is part of any cluster become anomaly."},{"metadata":{"trusted":true},"cell_type":"code","source":"data = df[['price_usd', 'srch_booking_window', 'srch_saturday_night_bool']]\nn_cluster = range(1, 20)\nkmeans = [KMeans(n_clusters=i).fit(data) for i in n_cluster]\nscores = [kmeans[i].score(data) for i in range(len(kmeans))]\n\nfig, ax = plt.subplots(figsize=(20,6))\nax.plot(n_cluster, scores)\nplt.xlabel('Number of Clusters', fontname=\"Times New Roman\",fontweight=\"bold\")\nplt.ylabel('Score',fontname=\"Times New Roman\",fontweight=\"bold\")\nplt.title(\"Elbow Curve\",fontname=\"Times New Roman\",fontweight=\"bold\")\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **We can see that after 10 Cluster we are not seeing any more variance so we can train model with 10 Cluster now**"},{"metadata":{"trusted":true},"cell_type":"code","source":"X = df[['price_usd', 'srch_booking_window', 'srch_saturday_night_bool']]\nX = X.reset_index(drop=True)\nkm = KMeans(n_clusters=10)\nkm.fit(X)\nkm.predict(X)\nlabels = km.labels_\n#Plotting\nfig = plt.figure(1, figsize=(12,12))\nax = Axes3D(fig, rect=[0, 0, 0.95, 1], elev=49, azim=140)\nax.scatter(X.iloc[:,0], X.iloc[:,1], X.iloc[:,2], c=labels.astype(np.float), edgecolor=\"r\")\nax.set_xlabel(\"price_usd\")\nax.set_ylabel(\"srch_booking_window\")\nax.set_zlabel(\"srch_saturday_night_bool\")\nplt.title(\"K Means Clustering for Anomaly Detection\", fontsize=20, fontweight=\"bold\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = df[['price_usd', 'srch_booking_window', 'srch_saturday_night_bool']]\nX = data.values\nX_std = StandardScaler().fit_transform(X)\nmean_vec = np.mean(X_std, axis=0)\ncov_mat = np.cov(X_std.T)\neig_vals, eig_vecs = np.linalg.eig(cov_mat)\neig_pairs = [ (np.abs(eig_vals[i]),eig_vecs[:,i]) for i in range(len(eig_vals))]\neig_pairs.sort(key = lambda x: x[0], reverse= True)\ntot = sum(eig_vals)\nvar_exp = [(i/tot)*100 for i in sorted(eig_vals, reverse=True)] # Individual explained variance\ncum_var_exp = np.cumsum(var_exp) # Cumulative explained variance\n\nplt.figure(figsize=(20, 6))\nplt.bar(range(len(var_exp)), var_exp, alpha=0.3, align='center', label='individual explained variance', color = 'g')\nplt.step(range(len(cum_var_exp)), cum_var_exp, where='mid',label='cumulative explained variance')\nplt.ylabel('Explained variance ratio')\nplt.xlabel('Principal components')\nplt.legend(loc='best')\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* You can see that first component and Seocnd component is contain 80% Explained variance**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Take useful feature and standardize them\ndata = df[['price_usd', 'srch_booking_window', 'srch_saturday_night_bool']]\nX_std = StandardScaler().fit_transform(X)\ndata = pd.DataFrame(X_std)\n# reduce to 2 important features\npca = PCA(n_components=2)\ndata = pca.fit_transform(data)\n# standardize these 2 new features\nscaler = StandardScaler()\nnp_scaled = scaler.fit_transform(data)\ndata = pd.DataFrame(np_scaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kmeans = [KMeans(n_clusters=i).fit(data) for i in n_cluster]\ndf['cluster'] = kmeans[9].predict(data)\ndf.index = data.index\ndf['principal_feature1'] = data[0]\ndf['principal_feature2'] = data[1]\ndf['cluster'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot the different clusters with the 2 main features\nfig, ax = plt.subplots(figsize=(10,6))\ncolors = {0:'red', 1:'blue', 2:'green', 3:'pink', 4:'black', 5:'orange', 6:'cyan', 7:'yellow', 8:'brown', 9:'purple', 10:'white', 11: 'grey'}\nax.scatter(df['principal_feature1'], df['principal_feature2'], c=df[\"cluster\"].apply(lambda x: colors[x]))\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# return Series of distance between each point and its distance with the closest centroid\nimport sys\ndef getDistanceByPoint(data, model):\n    distance = pd.Series()\n    for i in range(0,len(data)):\n        Xa = np.array(data.loc[i])\n        Xb = model.cluster_centers_[model.labels_[i]-1]\n        distance.set_value(i, np.linalg.norm(Xa-Xb))\n    return distance\n\noutliers_fraction = 0.01\n# get the distance between each point and its nearest centroid. The biggest distances are considered as anomaly\ndistance = getDistanceByPoint(data, kmeans[9])\nnumber_of_outliers = int(outliers_fraction*len(distance))\nthreshold = distance.nlargest(number_of_outliers).min()\n# anomaly1 contain the anomaly result of the above method Cluster (0:normal, 1:anomaly) \ndf['anomaly1'] = (distance >= threshold).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(20,8))\ncolors = {0:'blue', 1:'red'}\nax.scatter(df['principal_feature1'], df['principal_feature2'], c=df[\"anomaly1\"].apply(lambda x: colors[x]))\nplt.xlabel('principal feature1')\nplt.ylabel('principal feature2')\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.anomaly1.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.sort_values('date_time')\ndf[\"date_time\"] = pd.to_datetime(df[\"date_time\"])\ndf['date_time_int'] = df.date_time.astype(np.int64)\nfig, ax = plt.subplots(figsize=(20,6))\n\na = df.loc[df['anomaly1'] == 1, ['date_time_int', 'price_usd']] #anomaly\n\nax.scatter(a['date_time_int'],a['price_usd'], color='red', label='Anomaly',s = 200)\nax.plot(df['date_time_int'], df['price_usd'], color='blue', label='Normal',linewidth=0.7)\nplt.xlabel('Date Time Integer')\nplt.ylabel('price in USD')\nplt.legend()\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = df.loc[df['anomaly1'] == 0, 'price_usd']\nb = df.loc[df['anomaly1'] == 1, 'price_usd']\n\nfig, axs = plt.subplots(figsize=(20,6))\naxs.hist([a,b], bins=32, stacked=True, color=['blue', 'red'])\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Isolation forest for Anomaly Detection"},{"metadata":{"trusted":true},"cell_type":"code","source":"import altair as alt\nalt.renderers.enable('default')\ndata = df[['price_usd', 'srch_booking_window', 'srch_saturday_night_bool']]\nscaler = StandardScaler()\nnp_scaled = scaler.fit_transform(data)\ndata = pd.DataFrame(np_scaled)\n# train isolation forest\nmodel =  IsolationForest(contamination=outliers_fraction)\nmodel.fit(data)\n\ndf['anomaly2'] = pd.Series(model.predict(data))\n# df['anomaly2'] = df['anomaly2'].map( {1: 0, -1: 1} )\n\nfig, ax = plt.subplots(figsize=(20,10))\n\na = df.loc[df['anomaly2'] == -1, ['date_time_int', 'price_usd']] #anomaly\n\nax.plot(df['date_time_int'], df['price_usd'], color='blue', label = 'Normal',linewidth=0.7)\nax.scatter(a['date_time_int'],a['price_usd'], color='red', label = 'Anomaly', s = 200)\nplt.legend()\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualisation of anomaly with avg price repartition\na = df.loc[df['anomaly2'] == 1, 'price_usd']\nb = df.loc[df['anomaly2'] == -1, 'price_usd']\n\nfig, axs = plt.subplots(figsize=(20,8))\naxs.hist([a,b], bins=32, stacked=True, color=['blue', 'red'])\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Support Vector Machine-Based Anomaly Detection\n* A support vector machine is another effective technique for detecting anomalies. A SVM is typically associated with supervised learning, but OneClassSVM can be used to identify anomalies as an unsupervised problems.\n\n### One class SVM\nAccording to the paper: Support Vector Method for Novelty Detection. SVMs are max-margin methods, i.e. they do not model a probability distribution. The idea of SVM for anomaly detection is to find a function that is positive for regions with high density of points, and negative for small densities.\n\n* Unsupervised Outlier Detection.\n* Estimate the support of a high-dimensional distribution.\n* The implementation is based on libsvm."},{"metadata":{"trusted":true},"cell_type":"code","source":"data = df[['price_usd', 'srch_booking_window', 'srch_saturday_night_bool']]\nscaler = StandardScaler()\nnp_scaled = scaler.fit_transform(data)\ndata = pd.DataFrame(np_scaled)\n# train oneclassSVM \nmodel = OneClassSVM(nu=outliers_fraction, kernel=\"rbf\", gamma=0.01)\nmodel.fit(data)\n \ndf['anomaly3'] = pd.Series(model.predict(data))\n# df['anomaly3'] = df['anomaly3'].map( {1: 0, -1: 1} )\nfig, ax = plt.subplots(figsize=(20,6))\n\na = df.loc[df['anomaly3'] == -1, ['date_time_int', 'price_usd']] #anomaly\n\nax.plot(df['date_time_int'], df['price_usd'], color='blue', label ='Normal', linewidth = 0.7)\nax.scatter(a['date_time_int'],a['price_usd'], color='red', label = 'Anomaly', s = 100)\nplt.legend()\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"a = df.loc[df['anomaly3'] == 1, 'price_usd']\nb = df.loc[df['anomaly3'] == -1, 'price_usd']\n\nfig, axs = plt.subplots(figsize=(20,6))\naxs.hist([a,b], bins=32, stacked=True, color=['blue', 'red'])\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Anomaly Detection using Gaussian Distribution\n> * Gaussian distribution is also called **normal distribution**. We will be using the Gaussian distribution to develop an anomaly detection algorithm, that is, we’ll assume that our data are normally distributed. This’s an assumption that cannot hold true for all data sets, yet when it does, it proves an effective method for spotting outliers.\n\n> * Scikit-Learn’s **`covariance.EllipticEnvelope`** is a function that tries to figure out the key parameters of our data’s general distribution by assuming that our entire data is an expression of an underlying multivariate Gaussian distribution."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_class0 = df.loc[df['srch_saturday_night_bool'] == 0, 'price_usd']\ndf_class1 = df.loc[df['srch_saturday_night_bool'] == 1, 'price_usd']\n\nfig, axs = plt.subplots(1,2, figsize= (20,5))\ndf_class0.hist(ax=axs[0], bins=30)\ndf_class1.hist(ax=axs[1], bins=30)\naxs[0].set_title(\"Non Saturday Night\")\naxs[1].set_title(\"Saturday Night\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"envelope =  EllipticEnvelope(contamination = outliers_fraction) \nX_train = df_class0.values.reshape(-1,1)\nenvelope.fit(X_train)\ndf_class0 = pd.DataFrame(df_class0)\ndf_class0['deviation'] = envelope.decision_function(X_train)\ndf_class0['anomaly'] = envelope.predict(X_train)\n\nenvelope =  EllipticEnvelope(contamination = outliers_fraction) \nX_train = df_class1.values.reshape(-1,1)\nenvelope.fit(X_train)\ndf_class1 = pd.DataFrame(df_class1)\ndf_class1['deviation'] = envelope.decision_function(X_train)\ndf_class1['anomaly'] = envelope.predict(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot the price repartition by categories with anomalies\na0 = df_class0.loc[df_class0['anomaly'] == 1, 'price_usd']\nb0 = df_class0.loc[df_class0['anomaly'] == -1, 'price_usd']\n\na2 = df_class1.loc[df_class1['anomaly'] == 1, 'price_usd']\nb2 = df_class1.loc[df_class1['anomaly'] == -1, 'price_usd']\n\nfig, axs = plt.subplots(1,2, figsize= (20,5))\naxs[0].hist([a0,b0], bins=32, stacked=True, color=['blue', 'red'])\naxs[1].hist([a2,b2], bins=32, stacked=True, color=['blue', 'red'])\naxs[0].set_title(\"Search Non Saturday Night\")\naxs[1].set_title(\"Search Saturday Night\")\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add the data to the main \ndf_class = pd.concat([df_class0, df_class1])\ndf['anomaly5'] = df_class['anomaly']\n# df['anomaly5'] = np.array(df['anomaly22'] == -1).astype(int)\nfig, ax = plt.subplots(figsize=(20, 6))\na = df.loc[df['anomaly5'] == -1, ('date_time_int', 'price_usd')] #anomaly\nax.plot(df['date_time_int'], df['price_usd'], color='blue', label='Normal', linewidth = 0.7)\nax.scatter(a['date_time_int'],a['price_usd'], color='red', label='Anomaly', s = 100)\nplt.legend()\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***References*** : **https://towardsdatascience.com/time-series-of-price-anomaly-detection-13586cd5ff46**\n\n### Thanks for Reading...!!!\n\n***Note : This Notebook just made for practice.If I had done any mistake please provide feedback.***"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}