{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# PACKAGES AND LIBRARIES","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom warnings import filterwarnings\nfrom mpl_toolkits.mplot3d import Axes3D\nimport statsmodels.api as sm\nimport missingno as msno\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import scale\nfrom sklearn.neighbors import LocalOutlierFactor\nfrom scipy.stats import levene\nfrom scipy.stats import shapiro\nfrom scipy.stats.stats import pearsonr\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split, cross_val_score, cross_val_predict\nfrom sklearn.preprocessing import scale\nfrom sklearn.model_selection import ShuffleSplit, GridSearchCV\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn import model_selection\nfrom sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.ensemble import BaggingRegressor\nfrom sklearn.svm import SVR\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.neural_network import MLPRegressor\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.cross_decomposition import PLSRegression\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import RidgeCV\nfrom sklearn.linear_model import Lasso\nfrom sklearn.linear_model import LassoCV\nfrom sklearn.linear_model import ElasticNet\nfrom sklearn.linear_model import ElasticNetCV\nfrom sklearn import linear_model\nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\nfrom sklearn.ensemble import GradientBoostingRegressor, GradientBoostingClassifier\nimport xgboost as xgb\nfrom xgboost import XGBRegressor, XGBClassifier\nfrom lightgbm import LGBMRegressor, LGBMClassifier\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC\nfrom sklearn import tree\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report, roc_auc_score, roc_curve","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### WARNINGS ","metadata":{}},{"cell_type":"code","source":"filterwarnings(\"ignore\", category=DeprecationWarning) \nfilterwarnings(\"ignore\", category=FutureWarning) \nfilterwarnings(\"ignore\", category=UserWarning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATA","metadata":{}},{"cell_type":"code","source":"DigitsTrain = pd.read_csv(\"../input/digit-recognizer/train.csv\") # train main","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DigitsTest = pd.read_csv(\"../input/digit-recognizer/test.csv\") # test main","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = DigitsTrain.copy() # to protect train main data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataTest = DigitsTest.copy() # to protect test main data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EXPLORATORY DATA ANALYSIS","metadata":{}},{"cell_type":"markdown","source":"#### TRAIN DATA","metadata":{}},{"cell_type":"code","source":"print(data.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.columns.value_counts().sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.info())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.describe().T)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(data)) # checking type","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.corr()) # checking correlation\n# it is not suitable for this data, but you should use and try for others","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.cov()) # checking covariance\n# it is not suitable for this data, but you should use and try for others","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.groupby([\"label\"])[\"pixel0\"].mean()) # example for groupby\n# it is not suitable for this data, but you should use and try for others","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.groupby([\"label\"])[\"pixel783\"].mean()) # example for groupby\n# it is not suitable for this data, but you should use and try for others","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maxPixel783 = data[data[\"pixel783\"] == data[\"pixel783\"].max()] # example for checking max\n# it is not suitable for this data, but you should use and try for others\nprint(maxPixel783.index.value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.isnull().all().sum()) # checking all missing values for rows and columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.isnull().any().sum()) # checking any missing values for rows and columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.duplicated().sum()) # checking duplicated","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### TEST DATA","metadata":{}},{"cell_type":"code","source":"print(dataTest.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.info())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.describe().T)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.corr())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.cov())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.columns.value_counts().sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.isnull().all().sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.isnull().any().sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.duplicated().sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SPLITTING TRAIN AND TEST FOR TRAINING","metadata":{}},{"cell_type":"code","source":"x = data.drop(\"label\",axis=1) # features\ny = data[\"label\"] # prediction target","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xTrain,xTest,yTrain,yTest = train_test_split(x,y,test_size=0.2,random_state=42) # for testing our models","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = MinMaxScaler()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xTrain = scaler.fit_transform(xTrain)\nxTest = scaler.fit_transform(xTest)\n# for faster process\n# you can also use StandartScaler","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(xTrain.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(xTest.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# VISUALIZATION","metadata":{}},{"cell_type":"code","source":"figure = plt.figure(figsize=(15,8))\nsns.countplot(data[\"label\"],color=\"black\")\n# we see how many numbers are in the data\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0,5):\n    image = x.iloc[i]\n    image = np.array(image,dtype='uint8')\n    # we need to transform it to numpy array for visualizing\n    image = image.reshape((28,28))\n    # we need to reshape for visualizing because of pixels\n    plt.imshow(image,cmap=\"gray\")\n    plt.axis(\"off\")\n    plt.show()\n# visualizing numbers in data by using pixel values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODELS","metadata":{}},{"cell_type":"markdown","source":"#### CLASSIFIER","metadata":{}},{"cell_type":"code","source":"# models training\nlj = LogisticRegression(solver=\"liblinear\").fit(xTrain,yTrain)\ngnb = GaussianNB().fit(xTrain,yTrain)\nknnc = KNeighborsClassifier().fit(xTrain,yTrain)\ncartc = DecisionTreeClassifier(random_state=42).fit(xTrain,yTrain)\nrfc = RandomForestClassifier(random_state=42,verbose=False).fit(xTrain,yTrain)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelsc = [lj,gnb,knnc,cartc,rfc]\n# we will try 5 models for example\n# if you want you can try all of them","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# models' accuracy scores\nfor model in modelsc:\n    name = model.__class__.__name__\n    predict = model.predict(xTest)\n    R2CV = cross_val_score(model,xTest,yTest,cv=10,verbose=False).mean()\n    error = -cross_val_score(model,xTest,yTest,cv=10,scoring=\"neg_mean_squared_error\",verbose=False).mean()\n    print(name + \": \")\n    print(\"-\" * 10)\n    print(\"ACC-->\",accuracy_score(yTest,predict))\n    print(\"R2CV-->\",R2CV)\n    print(\"MEAN SQUARED ERROR-->\",np.sqrt(error))\n    print(\"-\" * 30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"as we see, the best is Random Forest for each of 5 models","metadata":{}},{"cell_type":"markdown","source":"#### PREDICTION","metadata":{}},{"cell_type":"code","source":"testlabel = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" we use \"dataTest\" for ANN, that is why we need to define again","metadata":{}},{"cell_type":"code","source":"predictC = rfc.predict(testlabel)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(predictC[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checking same value\nimageT = testlabel.iloc[0]\nimageT = np.array(imageT,dtype='uint8')\n# we need to transform it to numpy array for visualizing\nimageT = imageT.reshape((28,28))\n# we need to reshape for visualizing because of pixels\nplt.imshow(imageT,cmap=\"gray\")\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"IT'S TRUE","metadata":{}},{"cell_type":"markdown","source":"#### SUBMISSION","metadata":{}},{"cell_type":"code","source":"print(predictC.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imageIDC = [h+1 for h in range(len(predictClass))]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(imageIDC)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissionClass = pd.DataFrame()\nsubmissionClass[\"ImageId\"] = imageIDC\nsubmissionClass[\"Label\"] = predictC","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissionC.to_csv('submissionClass.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ANN MODEL","metadata":{}},{"cell_type":"markdown","source":"#### LIBRARY ","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.layers import Dropout\nfrom tensorflow.keras.layers import Flatten\nimport tensorflow as tf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### TRAIN","metadata":{}},{"cell_type":"code","source":"xN = data.drop(\"label\",axis=1)\nyN = data[\"label\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xN = np.array(xN)\nxN = xN.reshape(42000, 28, 28)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(xN.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CREATING","metadata":{}},{"cell_type":"code","source":"ANNmodel = tf.keras.models.Sequential([\n  # inputs\n  tf.keras.layers.experimental.preprocessing.Rescaling(1./255),\n  tf.keras.layers.Flatten(input_shape=(28, 28)),\n  # hiddens\n  tf.keras.layers.Dense(128, activation='relu'),\n  tf.keras.layers.Dropout(0.2),\n  # output\n  tf.keras.layers.Dense(10)\n])\n\nloss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\nANNmodel.compile(optimizer='adam', loss=loss_fn, metrics=['accuracy'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\n\nMainModel = ANNmodel.fit(xN, yN, validation_split=0.2, epochs=30, callbacks=[callback])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(ANNmodel.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = pd.DataFrame(ANNmodel.history.history)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(loss.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss.plot()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(MainModel.history[\"accuracy\"])\nplt.plot(MainModel.history[\"val_accuracy\"])\nplt.xlabel(\"EPOCH\")\nplt.ylabel(\"ACC\")\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### PREDICTION","metadata":{}},{"cell_type":"code","source":"dataTest = np.array(dataTest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(dataTest.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataTest = dataTest.reshape(28000,28,28)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictANN = ANNmodel.predict(dataTest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(dataTest[0])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = [np.argmax(i) for i in predictANN]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checking same value\nprint(p[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"IT'S TRUE","metadata":{}},{"cell_type":"markdown","source":"#### SUBMISSION","metadata":{}},{"cell_type":"code","source":"imageID = [x+1 for x in range(len(p))]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission[\"ImageId\"] = imageID\nsubmission[\"Label\"] = p","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submissionANN.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}