{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0,"cells":[{"metadata":{"_cell_guid":"0237aa84-eab1-682e-f7d1-9e07cd47161c","_active":false,"collapsed":false},"source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\n\nimport plotly.graph_objs as go\nimport plotly.tools as tls\nimport seaborn as sns\nimport matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\nimport matplotlib\n%matplotlib inline\nfrom sklearn.manifold import TSNE\nfrom sklearn.decomposition import PCA\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"8ca08e1f-bfe3-77db-e6ca-b18919bc1946","_active":false,"collapsed":false},"source":"train=pd.read_csv('../input/train.csv')\ntrain.head()","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"583a5a2b-245d-f1f1-51d2-1c20a1a18a7a","_active":false,"collapsed":false},"source":"print(train.shape)","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"fbb43789-0548-e2e9-5081-27aaaf8f7667","_active":false,"collapsed":false},"source":"f,ax=plt.subplots(figsize=(5.1,5.1))\nplt.title('Correlation plot of a 100 columns in the MNIST dataset')\nsns.heatmap(train.ix[:,100:200].astype(float).corr(),linewidths=0,square=True,cmap='cubehelix',xticklabels=False,yticklabels=False,annot=True)\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"10cb11f6-c777-3ff6-077a-c5801e8ad57c","_active":false,"collapsed":false},"source":"target=train['label']\ntrain=train.drop(\"label\",axis=1)\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"1f31e19b-7e6a-6642-f12d-3b80a0196d7d","_active":false,"collapsed":false},"source":"from sklearn.preprocessing import StandardScaler\nX=train.values\nX_std=StandardScaler().fit_transform(X)\n#print(\"x=\",X)\n#print(\"x_std=\",X_std)\nmean_vec=np.mean(X_std,axis=0)\n#print(\"mean_vec=\",mean_vec)\ncov_mat=np.cov(X_std.T)\nprint(\"cov_mat=\",cov_mat)\neig_vals,eig_vecs=np.linalg.eig(cov_mat)\nprint(\"eig_vals=\",eig_vals)\nprint(\"eig_vecs=\",eig_vecs)\neig_pairs=[(np.abs(eig_vals[i]),eig_vecs[:,i]) for i in range(len(eig_vals))]\nprint(\"eig_pairs=\",eig_pairs)\neig_pairs.sort(key=lambda x:x[0],reverse=True)\nprint(\"eig_pairs=\",eig_pairs)\ntot=sum(eig_vals)\nprint(\"tot=\",tot)\nvar_exp=[(i/tot)*100 for i in sorted(eig_vals,reverse=True)]\nprint(\"var_exp=\",var_exp)\ncum_var_exp=np.cumsum(var_exp)\nprint(\"cum_var_exp=\",cum_var_exp)","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"efa93fbd-d364-4c53-630d-e7c288c924b8","_active":false,"collapsed":false},"source":"trace1=go.Scatter(\n    x=list(range(784)),\n    y=cum_var_exp,\n    mode='lines+markers',\n    name=\"Cumulative Explained Variance\",\n    hoverinfo=cum_var_exp,\n    line=dict(\n        shape='spline',\n        color='goldenrod'\n    )\n    \n)\ntrace2=go.Scatter(\n    x=list(range(784)),\n    y=var_exp,\n    mode='lines+markers',\n    name=\"Individual Explained Variance\",\n    hoverinfo=var_exp,\n    line=dict(\n        shape='linear',\n        color='black'\n    )\n)\n\nfig=tls.make_subplots(insets=[{\"cell\":(1,1),\"l\":0.7,\"b\":0.5}],print_grid=True)\nfig.append_trace(trace1,1,1)\nfig.append_trace(trace2,1,1)\nfig.layout.title='Explained Variance plots -Full and zoomed in'\nfig.layout.xaxis=dict(range=[0,80],title='Feature columns')\nfig.layout.yaxis=dict(range=[0,60],title='Explained Variance')\nfig['data']+=[go.Scatter(x=list(range(784)),y=cum_var_exp,xaxis=\"x2\",yaxis=\"y2\",name=\"Cumulative Explained Variance\")]\nfig['data']+=[go.Scatter(x=list(range(784)),y=var_exp,xaxis=\"x2\",yaxis=\"y2\",name=\"Individual Explained Variance\")]\npy.iplot(fig,filename='inset example')\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"35c838be-4708-38be-efd3-7b8e423c024c","_active":false,"collapsed":false},"source":"n_components=30\npca=PCA(n_components=n_components).fit(train.values)\neigenvalues=pca.components_.reshape(n_components,28,28)\neigenvalues=pca.components_","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"408500cf-860c-d1e5-c8b4-56ada5b08c6a","_active":false,"collapsed":false},"source":"n_row=4\nn_col=7\nplot.figure(figsize=(8,7))\nfor i in list(range(n_row*n_col)):\n    offset=0\n    plot.subplot(n_row,n_col,i+1)\n    plt.imshow(eigenvalues[i].reshape(28,28),cmap='jet')\n    title_text='Eigenvalue'+str(i+1)\n    plt.title(title_text,size=6.5)\n    plt.xticks()\n    plt.yticks()\nplt.show()\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"b67ddc91-c7eb-de91-360b-ab10ab850e26","_active":false,"collapsed":false},"source":"plt.figure(figsize=(7,7))\nfor digit_num in range(0,70):\n    plt.subplot(7,10,digit_num+1)\n    grid_data=train.iloc[digit_num].as_matrix().reshape(28,28)\n    plt.imshow(grid_data,interpolation=\"none\",cmap=\"afmhot\")\n    plt.xticks([])\n    plt.yticks()\nplt.tight_layout()\n\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"00142dd0-175a-a160-6adb-a154199411b5","_active":false,"collapsed":false},"source":"del X\nX=train[:2000].values\ndel train\nX_std=StandardScaler().fit_transform(X)\npca=PCA(n_components=5)\npca.fit(X_std)\nX_5d=pca.transform(X_std)\nTarget=target[:2000]","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"31d20270-9544-2d21-3ec1-7a3b8f961b7a","_active":false,"collapsed":false},"source":"trace0=go.Scatter(\n    x=X_5d[:,0],\n    y=X_5d[:,1],\n    name=Target,\n    hoveron=Target,\n    mode='markers',\n    text=Target,\n    showlegend=False,\n    marker=dict(\n        size=8,\n        color=Target,\n        colorscale='Jet',\n        showscale=False,\n        line=dict(\n            width=2,\n            color='rgb(255,255,255)'\n        )\n    )\n)\n\ndata=[trace0]\n\nlayout=go.Layout(\n    title='Principal Component Analysis(PCA)',\n    hovermode='closest',\n    xaxis=dict(\n        title='First Principal Component',\n        ticklen=5,\n        zeroline=False,\n        gridwidth=2,\n    ),\n    yaxis=dict(\n        title='Second Principal Component',\n        ticklen=5,\n        gridwidth=2,\n    ),\n    showlegend=True\n)\n\nfig=dict(data=data,layout=layout)\npy.iplot(fig,filename='styled-scatter')\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"0d23b5cf-ea8f-9270-8fa7-188790599ccc","_active":false,"collapsed":false},"source":"from sklearn.cluster import KMeans\nkmeans=KMeans(n_clusters=9)\nX_clustered=kmeans.fit_predict(X_5d)\n\ntrace_Kmeans=go.Scatter(x=X_5d[:,0],y=X_5d[:,1],mode=\"markers\",\n                       showlegend=False,\n                       marker=dict(\n                           size=8,\n                           color=X_clustered,\n                           colorscale='Portland',\n                           showscale=False,\n                           line=dict(\n                               width=2,\n                               color='rgb(255,255,255)'\n                           )\n                       ))\nlayout=go.Layout(\n    title='KMeans Clustering',\n    hovermode='closest',\n    xaxis=dict(\n        title=\"First Principal Component\",\n            ticklen=5,\n            zeroline=False,\n            gridwidth=2,\n        ),yaxis=dict(\n            title='Second Principal Component',\n            ticklen=5,\n            gridwidth=2,\n        ),\n        showlegend=True\n    )\ndata=[trace_Kmeans]\nfig1=dict(data=data,layout=layout)\npy.iplot(fig1,filename=\"svm\")","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"667f7c08-70ed-5451-914f-769c38dbf484","_active":false,"collapsed":false},"source":"from IPython.display import display,Math,Latex\nlda=LDA(n_components=5)\nX_LDA_2D=lda.fit_transform(X_std,Target.values)","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"66a8b401-3882-ee98-df08-3eb01d90ac7f","_active":false,"collapsed":false},"source":"traceLDA=go.Scatter(\n    x=X_LDA_2D[:,0],\n    y=X_LDA_2d[:,1],\n    name=Target,\n    mode='markers',\n    text=Target,\n    showlegent=True,\n    marker=dict(\n        size=8,\n        color=Target,\n        colorscale='Jet',\n        showscale=False,\n        line=dict(\n            width=2,\n            color='rgb(255,255,255)'\n        ),\n        opacity=0.8\n    )\n)\ndata=[traceLDA]\nlayout=go.Layout(\n    title='Linear Discriminant Analysis(LDA)',\n    hovermode='closest',\n    xaxis=dict(\n        title='First Linear Discriminant',\n        ticklen=5,\n        zeroline=False,\n        gridwidth=2,\n    ),\n    yaxis=dict(\n        title='Second Linear Discriminant',\n        ticklen=5,\n        gridwidth=2,\n    ),\n    showlegend=False\n)\nfig=dict(data=data,layout=layout)\npy.iplot(fig,filename='styled-scatter')","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"fcf1cea1-2c12-c019-5550-e83d013f8dc7","_active":false,"collapsed":false},"source":"tsne=TSNE()\ntsne_results=tsne.fit_transform(X_std)\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"f1031c73-9424-bb5a-0cf0-b5cbecf7279a","_active":false,"collapsed":false},"source":"traceTSNE=go.Scatter(\n    x=tsne_result[:,0],\n    y=tsne_result[:,1],\n    name=Target,\n    hoveron=Target,\n    mode='markers',\n    text=Target,\n    showlegend=True,\n    marker=dict(\n        size=8,\n        color=Target,\n        colorscale='Jet',\n        showscale=False,\n        line=dict(\n            width=2,\n            color='rgb(255,255,255)'\n        ),\n        opacity=0.8\n    )\n)\ndata=[traceTSNE]\nlayout=dict(title='TSNE (T-Distributed Stochastic Neighbour Embedding)',\n             hovermode='closest',\n            yaxis=dict(zeroline=False),\n            xaxis=dict(zeroline=False),\n            showlegend=False,\n           )\nfig=dict(data=data,layout=layout)\npy.iplot(fig,filename='styled-scatter')","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"33dd23ee-dcc6-48bc-4074-88fb6c798b01","_active":false,"collapsed":false},"source":"import numpy\nfrom sklearn.decomposition import PCA\nfrom sklearn.svm import SVC\nCOMPONENT_NUM=35\nprint('Read training data ...')\nwith open('../input/train.csv','r') as reader:\n    reader.readline()\n    train_label=[]\n    train_data=[]\n    for line in reader.readlines():\n        data=list(map(int,line.rstrip().split(',')))\n        train_label.append(data[0])\n        train_data.append(data[1:])\nprint('Loaded '+str(len(train_label)))\n\n\nprint('Reduction ....')\n\ntrain_label=numpy.array(train_label)\ntrain_data=numpy.array(train_data)\npca=PCA(n_components=COMPONENT_NUM,whiten=True)\npca.fit(train_data)\ntrain_data=pca.transform(train_data)\n\nprint('Train SVM ...')\nsvc=SVC()\nsvc.fit(train_data,train_label)\nprint('Read testing data ....')\nwith open('../input/test.csv','r') as reader:\n    reader.readline()\n    test_data=[]\n    for line in reader.readlines():\n        pixels=lit(map(int,line.rstrip().split(',')))\n        test_data.append(pixels)\n        \nprint(\"Loaded \"+str(len(test_data)))\n\nprint(\"predicting ...\")\ntest_data=numpy.array(test_data)\ntest_data=pca.transform(test_data)\npredict=svc.predict(test_data)\n\nprint('Saving ....')\nwith open('predict.csv','w') as writer:\n    writer.write(\"ImageId,Label\\n\")\n    count=0\n    for p in predict:\n        count+=1\n        writer.write(str(count),\",\"+str(p)+\"\\n\")","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"b68b29b1-4f10-604c-599b-a4515fb49af1","_active":false,"collapsed":false},"source":"import numpy as np\nimport matplotlib.pyplot as plt\nfrom collections import Counter\nimport time\n%matplotlib inline\nplt.rcParams['figure.figsize']=(10,8)\nplt.rcParams['image.interpolation']='nearest'\nplt.rcParams['image.cmap']='gray'\n\ndef load_data(data_dir):\n    train_data=open(data_dir+\"train.csv\").read()\n    train_data=train_data.split(\"\\n\")[1:-1]\n    train_data=[i.split(\",\") for i in train_data]\n    X_train=np.array([[int(i[j]) for j in range(1,len(i))] for i in train_data])\n    y_train=np.array([int(i[0]) for i in train_data])\n    \n    test_data=open(data_dir+\"test.csv\").read()\n    test_data=test_data.split(\"\\n\")[1:-1]\n    test_data=[i.split(\",\") for i in test_data]\n    X_test=np.array([[int(i[j]) for j in range(0,len(j))] for i in test_data])\n    return X_train,y_train,X_test\n\n\nclass simple_knn():\n    def __init__(self):\n        pass\n    def train(self,X,y):\n        self.X_train=X\n        self.y_traiin=y\n    \n    def predict(self,X,k=1):\n        dists=self.compute_distances(X)\n        num_test=dists.shape[0]\n        y_pred=np.zeros(num_test)\n        \n        for i in range(num_test):\n            k_closest_y=[]\n            labels=self.y_train[np.argsort(dists[i,:])].flatten()\n            k_closest_y=labels[:k]\n            c=Counter(k_closest_y)\n            y_pred[i]=c.most_common(1)[0][0]\n            \n        return y_pred\n    def compute_distances(self,X):\n        num_test=X.shape[0]\n        num_train=self.X_train.shape[0]\n        dot_pro=np.dot(X,self.X_train.T)\n        sum_square_test=np.square(X).sum(axis=1)\n        sum_square_train=np.square(self.X_train).sum(axis=1)\n        dists=np.sqrt(-2*dot_pro+sum_square_train+np.matrix(sum_square_test).T)\n        \n        return dists","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"dbc23055-4ffe-bf71-cfab-c9040eea08ff","_active":false,"collapsed":false},"source":"data_dir=\"../input/\"\nX_train,y_train,X_test=load_data(data_dir)\nprint(X_train.shape,y_train.shape,X_test.shape)\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"368219bc-e53a-3008-3ac0-b49a6e7b36b5","_active":false,"collapsed":false},"source":"classes=[\"0\",\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\"]\nnum_classes=len(classes)\nsamples=8\nfor y,cls in enumerate(classes):\n    idxs=np.nonzero([i==y for i in y_train])\n    idxs=np.random.choice(idxs[0],samples,replace=False)\n    for i ,idx in enumerate(idxs):\n        plt_idx=i*num_classes+y+1\n        plt.subplot(samples,num_classes,plt_idx)\n        plt.imshow(X_train[idx].reshape((28,28)))\n        plt.axis(\"off\")\n        if i==0:\n            plt.title(cls)\n            \n            \nplt.show()","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"9978a088-b98c-d325-a8b8-ae7d7a7e88a2","_active":false,"collapsed":false},"source":"plt.imshow(X_test[2311].reshape((28,28)))","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"b5f44f53-8db8-3ebf-ec5f-937f5bff1d8a","_active":false,"collapsed":false},"source":"batch_size=2000\nk=1\nclassifier=simple_knn()\nclassifier.train(X_train,y_train)\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"4098d8e2-e329-a16c-ce69-272ba091465a","_active":false,"collapsed":false},"source":"predictions=[]\nfor i in range(int(len(X_test)/(2*batch_size))):\n    tic=time.time()\n    predts=classifier.predict(X_test[i*batch_size:(i+1)*batch_size],k)\n    toc=time.time()\n    predictions=predictions+list(predts)","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"88662225-92fd-a14b-88f8-e1fdd07b364f","_active":false,"collapsed":false},"source":"import pandas as pd\nimport matplotlib.pyplot as plt,matplotlib.image as mpimg\nfrom sklearn.model_selection import train_test_split\nfrom sklean import svm\n%matplotlib inline","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"e90e7169-186d-7ba1-4ef6-18b11e3f63b1","_active":false,"collapsed":false},"source":"labeled_images=pd.read_csv('../input/train.csv')\nimages=labeled_images.iloc[0:5000,1:]\nlabels=labeled_images.iloc[0:5000,1:]\ntrain_images,test_images,train_labels,test_labels=train_test_split(images,labels,train_size=0.8,random_state=0)\n","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"3d1b614e-bbe5-de7c-72dc-7e7e6e8ed795","_active":false,"collapsed":false},"source":"i=1\nimg=train_images.iloc[i].as_matrix()\nimg=img.reshape((28,28))\nplt.imshow(img,cmap='gray')\nplt.title(trian_labels.iloc[i,0])","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"9bf3af4a-f4dc-d80f-a9e3-627182a571d0","_active":false,"collapsed":false},"source":"plt.hist(train_images.iloc[i])","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"},{"metadata":{"_cell_guid":"249aae52-9e3c-cca4-0824-3d2e00e2db1d","_active":true,"collapsed":false},"source":"clf=svm.SVC()\nclf.fit(train_images,train_labels.vlaues_ravel())\nclf.score(test_images,test_labels)","execution_count":null,"cell_type":"code","outputs":[],"execution_state":"idle"}]}