{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":15520,"sourceType":"datasetVersion","datasetId":11167}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Importing libraries","metadata":{}},{"cell_type":"markdown","source":"I will be only using the following libraries only even though it is possible to write them from scratch too but my purpose of writing this notebook is to implement a Random Forest Classifier from scratch.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:51:46.310636Z","iopub.execute_input":"2024-04-22T08:51:46.311115Z","iopub.status.idle":"2024-04-22T08:51:49.484359Z","shell.execute_reply.started":"2024-04-22T08:51:46.311074Z","shell.execute_reply":"2024-04-22T08:51:49.482766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Importing Data and Initial Analysis","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/mobile-price-classification/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:51:49.486973Z","iopub.execute_input":"2024-04-22T08:51:49.487673Z","iopub.status.idle":"2024-04-22T08:51:49.519459Z","shell.execute_reply.started":"2024-04-22T08:51:49.487628Z","shell.execute_reply":"2024-04-22T08:51:49.518125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# First 5 rows\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:51:49.521208Z","iopub.execute_input":"2024-04-22T08:51:49.521693Z","iopub.status.idle":"2024-04-22T08:51:49.559842Z","shell.execute_reply.started":"2024-04-22T08:51:49.521647Z","shell.execute_reply":"2024-04-22T08:51:49.558620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It appears there are 21 columns and we are going to use most of these columns to predict the Price range of the mobile.","metadata":{}},{"cell_type":"markdown","source":"Let me explain what each of these columns are so that we can understand what can actually affect the price of the mobile.\n* In this data:\n    * id: ID\n    * battery_power: Total energy a battery can store in one time measured in mAh\n    * blue: Has bluetooth or not\n    * clock_speed: Speed at which microprocessor executes instructions\n    * dual_sim: Has dual sim support or not\n    * fc: Front Camera mega pixels\n    * four_g: Has 4G or not\n    * int_memory: Internal Memory in Gigabytes\n    * m_dep: Mobile Depth in cm\n    * mobile_wt: Weight of mobile phone\n    * n_cores: Number of cores of processor\n    * pc: Primary Camera mega pixels\n    * px_height: Pixel Resolution Height\n    * px_width: Pixel Resolution Width\n    * ram: Random Access Memory in Megabytes\n    * sc_h: Screen Height of mobile in cm\n    * sc_w: Screen Width of mobile in cm\n    * talk_time: Longest time that a single battery charge will last when you are\n    * three_g: Has 3G or not\n    * touch_screen: Has touch screen or not\n    * wifi: Has wifi or not\n","metadata":{}},{"cell_type":"code","source":"# Shape of dataset\nprint(\"Shape of the Dataset is:\", df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:51:49.562881Z","iopub.execute_input":"2024-04-22T08:51:49.563655Z","iopub.status.idle":"2024-04-22T08:51:49.570308Z","shell.execute_reply.started":"2024-04-22T08:51:49.563611Z","shell.execute_reply":"2024-04-22T08:51:49.569229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check for any missing values.","metadata":{}},{"cell_type":"code","source":"# Missing Values\ndf.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:54.158761Z","iopub.execute_input":"2024-04-22T08:53:54.159231Z","iopub.status.idle":"2024-04-22T08:53:54.170324Z","shell.execute_reply.started":"2024-04-22T08:53:54.159198Z","shell.execute_reply":"2024-04-22T08:53:54.169026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Luckily there aren't any. Now let's find the unique values.","metadata":{}},{"cell_type":"code","source":"# Unique values count\ndict = {}\nfor i in list(df.columns):\n    dict[i] = df[i].value_counts().shape[0]\n\npd.DataFrame(dict, index=[\"Unique Values\"]).transpose()    ","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:54.567098Z","iopub.execute_input":"2024-04-22T08:53:54.567572Z","iopub.status.idle":"2024-04-22T08:53:54.598178Z","shell.execute_reply.started":"2024-04-22T08:53:54.567532Z","shell.execute_reply":"2024-04-22T08:53:54.596989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This shows is that there are some columns which are categorical and some which are continuous.","metadata":{}},{"cell_type":"code","source":"df.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:55.158536Z","iopub.execute_input":"2024-04-22T08:53:55.158973Z","iopub.status.idle":"2024-04-22T08:53:55.168652Z","shell.execute_reply.started":"2024-04-22T08:53:55.158941Z","shell.execute_reply":"2024-04-22T08:53:55.167407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe().transpose()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:55.412938Z","iopub.execute_input":"2024-04-22T08:53:55.413403Z","iopub.status.idle":"2024-04-22T08:53:55.480792Z","shell.execute_reply.started":"2024-04-22T08:53:55.413366Z","shell.execute_reply":"2024-04-22T08:53:55.479507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Data Visualization","metadata":{}},{"cell_type":"markdown","source":"### How is ram related to price_range?","metadata":{}},{"cell_type":"code","source":"sns.kdeplot(x='ram',hue='price_range',data=df, fill = True, alpha = 0.6, palette = \"crest\")","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:56.116549Z","iopub.execute_input":"2024-04-22T08:53:56.117770Z","iopub.status.idle":"2024-04-22T08:53:56.706466Z","shell.execute_reply.started":"2024-04-22T08:53:56.117727Z","shell.execute_reply":"2024-04-22T08:53:56.704953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How is Internal Memory related to Price Range?","metadata":{}},{"cell_type":"code","source":"sns.kdeplot(x = \"int_memory\", hue = \"price_range\", data = df, fill = True, palette = \"dark\")","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:56.708297Z","iopub.execute_input":"2024-04-22T08:53:56.709301Z","iopub.status.idle":"2024-04-22T08:53:57.088038Z","shell.execute_reply.started":"2024-04-22T08:53:56.709260Z","shell.execute_reply":"2024-04-22T08:53:57.086699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Percentage of phones which support 3G","metadata":{}},{"cell_type":"code","source":"labels = [\"3G-supported\",'Not supported']\nvalues=df['three_g'].value_counts().values","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:57.252180Z","iopub.execute_input":"2024-04-22T08:53:57.253404Z","iopub.status.idle":"2024-04-22T08:53:57.258837Z","shell.execute_reply.started":"2024-04-22T08:53:57.253360Z","shell.execute_reply":"2024-04-22T08:53:57.257896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig1, ax1 = plt.subplots()\nax1.pie(values, labels=labels, autopct='%1.1f%%',shadow=True,startangle=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:57.503558Z","iopub.execute_input":"2024-04-22T08:53:57.504068Z","iopub.status.idle":"2024-04-22T08:53:57.665783Z","shell.execute_reply.started":"2024-04-22T08:53:57.504032Z","shell.execute_reply":"2024-04-22T08:53:57.664173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Percentage of phones which support 4G","metadata":{}},{"cell_type":"code","source":"labels_4g = [\"4G-supported\",'Not supported']\nvalues_4g = df['four_g'].value_counts().values\nfig1, ax1 = plt.subplots()\nax1.pie(values_4g, labels=labels_4g, autopct='%1.1f%%', shadow=True, startangle=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:57.949612Z","iopub.execute_input":"2024-04-22T08:53:57.950054Z","iopub.status.idle":"2024-04-22T08:53:58.105715Z","shell.execute_reply.started":"2024-04-22T08:53:57.950023Z","shell.execute_reply":"2024-04-22T08:53:58.104124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How is Battery Power related to Price Range?","metadata":{}},{"cell_type":"code","source":"sns.boxplot(x=\"price_range\", y=\"battery_power\", data=df)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:58.604576Z","iopub.execute_input":"2024-04-22T08:53:58.605845Z","iopub.status.idle":"2024-04-22T08:53:58.885613Z","shell.execute_reply.started":"2024-04-22T08:53:58.605802Z","shell.execute_reply":"2024-04-22T08:53:58.884553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of Number of phones vs Camera MP of Front and Primary Camera","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\ndf['fc'].hist(alpha=0.7,color='green',label='Front camera')\ndf['pc'].hist(alpha=0.6,color='red',label='Primary camera')\nplt.legend()\nplt.xlabel('MegaPixels')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:53:59.404590Z","iopub.execute_input":"2024-04-22T08:53:59.405816Z","iopub.status.idle":"2024-04-22T08:53:59.828005Z","shell.execute_reply.started":"2024-04-22T08:53:59.405770Z","shell.execute_reply":"2024-04-22T08:53:59.826983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How is Mobile's Weight is related to the Price Range?","metadata":{}},{"cell_type":"code","source":"sns.jointplot(x='mobile_wt', y='price_range', data=df, kind='kde')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:00.783100Z","iopub.execute_input":"2024-04-22T08:54:00.783864Z","iopub.status.idle":"2024-04-22T08:54:02.989968Z","shell.execute_reply.started":"2024-04-22T08:54:00.783819Z","shell.execute_reply":"2024-04-22T08:54:02.988536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation Heatmap","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,15))\nsns.heatmap(df.corr(), annot=True, cmap=plt.cm.plasma)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:02.992295Z","iopub.execute_input":"2024-04-22T08:54:02.992851Z","iopub.status.idle":"2024-04-22T08:54:04.875671Z","shell.execute_reply.started":"2024-04-22T08:54:02.992801Z","shell.execute_reply":"2024-04-22T08:54:04.874242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Data Preprocessing","metadata":{}},{"cell_type":"code","source":"X_array = df.drop('price_range', axis=1)\ny_array = df['price_range']","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:04.878276Z","iopub.execute_input":"2024-04-22T08:54:04.878807Z","iopub.status.idle":"2024-04-22T08:54:04.887054Z","shell.execute_reply.started":"2024-04-22T08:54:04.878763Z","shell.execute_reply":"2024-04-22T08:54:04.885853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X_array.values","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:04.888829Z","iopub.execute_input":"2024-04-22T08:54:04.890003Z","iopub.status.idle":"2024-04-22T08:54:04.899541Z","shell.execute_reply.started":"2024-04-22T08:54:04.889949Z","shell.execute_reply":"2024-04-22T08:54:04.898123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = y_array.values","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:04.902382Z","iopub.execute_input":"2024-04-22T08:54:04.903198Z","iopub.status.idle":"2024-04-22T08:54:04.911641Z","shell.execute_reply.started":"2024-04-22T08:54:04.903150Z","shell.execute_reply":"2024-04-22T08:54:04.910256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I'll use the train_test_split code from my previous notebook.","metadata":{}},{"cell_type":"code","source":"def train_test_split(X, y, random_state=123, test_size=0.20):\n    \"\"\"\n    Splits the data into training and testing sets.\n\n    Parameters:\n        X (numpy.ndarray or pandas.DataFrame): Features array of shape (n_samples, n_features).\n        y (numpy.ndarray or pandas.Series): Target array of shape (n_samples,).\n        random_state (int): Seed for the random number generator. Default is 42.\n        test_size (float): Proportion of samples to include in the test set. Default is 0.2.\n\n    Returns:\n        Tuple[numpy.ndarray or pandas.DataFrame]: A tuple containing X_train, X_test, y_train, y_test.\n    \"\"\"\n    # Convert input to numpy arrays if pandas DataFrames or Series are provided\n    if isinstance(X, pd.DataFrame):\n        X = X.values\n    if isinstance(y, pd.Series):\n        y = y.values\n\n    # Ensure X and y have the same number of samples\n    assert X.shape[0] == y.shape[0], \"X and y must have the same number of samples.\"\n\n    # Get number of samples\n    n_samples = X.shape[0]\n\n    # Set the seed for the random number generator\n    np.random.seed(random_state)\n\n    # Shuffle the indices\n    shuffled_indices = np.random.permutation(np.arange(n_samples))\n\n    # Determine the size of the test set\n    test_size = int(n_samples * test_size)\n\n    # Split the indices into test and train\n    test_indices = shuffled_indices[:test_size]\n    train_indices = shuffled_indices[test_size:]\n\n    # Split the features and target arrays into test and train\n    X_train, X_test = X[train_indices], X[test_indices]\n    y_train, y_test = y[train_indices], y[test_indices]\n\n    return X_train, X_test, y_train, y_test","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:05.291648Z","iopub.execute_input":"2024-04-22T08:54:05.292987Z","iopub.status.idle":"2024-04-22T08:54:05.302824Z","shell.execute_reply.started":"2024-04-22T08:54:05.292940Z","shell.execute_reply":"2024-04-22T08:54:05.301470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=101)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:05.694570Z","iopub.execute_input":"2024-04-22T08:54:05.695495Z","iopub.status.idle":"2024-04-22T08:54:05.703308Z","shell.execute_reply.started":"2024-04-22T08:54:05.695448Z","shell.execute_reply":"2024-04-22T08:54:05.702080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Model Implementation\n\nI'm importing this as the **Random Forest** requires **Decision Tree** in its code.","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\ndtc = DecisionTreeClassifier()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:06.670844Z","iopub.execute_input":"2024-04-22T08:54:06.671295Z","iopub.status.idle":"2024-04-22T08:54:07.384520Z","shell.execute_reply.started":"2024-04-22T08:54:06.671263Z","shell.execute_reply":"2024-04-22T08:54:07.383340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Random Forest\n\n## Key Concepts\n\n### 1. Gini Impurity:\nGini impurity is a measure of the disorder or uncertainty in a set of labels. It quantifies the probability of incorrectly classifying an instance's label if it were randomly labeled according to the distribution of labels in the set.\n\nGini impurity $Gini(D)$ of a dataset $D$ with $K$ distinct classes is calculated using the following formula:\n$$\nGini(D) = 1 - \\sum_{i=1}^k p_i^2\n$$\nWhere $p_{i}$ is the probability of randomly selecting an instance of class $i$ from the dataset $D$.\n\n\n### 2. Bootstraping:\nBootstraping is the process of creating multiple bootstrap samples(data subsets sampled with replacement) from the original training data. Each bootstrap sample is used to grow a different decision tree in the random forest. This process introduces randomness, which helps reduce overfitting and improves the overall predictive accuracy of the ensemble model compared to a single decision tree. It also allows for estimating the generalization error of the random forest using out-of-bag(OOB) observations.\n\n\n### 3. OOB score(Out-of-Bag Score):\nThe OOB score is an internal estimate of the prediction error of a random forest model, calculated using the observations that were not included in the bootstrap samples (i.e., out-of-bag observations) used to build each decision tree.\n\nThe formula being implemented in the code is:\n$$\nOOB Error = 1 - \\frac{1}{n} * \\sum_{i=1}^{n} I(y_i == y_{pred(i)})\n$$\nWhere **n** is the total number of observations, **$y_i$** is the true target value for observation **i**, **$y_{pred(i)}$** is the prediction from the random forest for observation **i** using only trees where **i** was OOB, **$I(x)$** is an indicator function that returns 1 if $x$ is true, 0 otherwise.\n\n### 4. Most common label:\nThe most common label (or majority class) in a random forest, or any classification problem, refers to the class or label that occurs most frequently in the target variable (y) of the training data.","metadata":{}},{"cell_type":"markdown","source":"### How the algorithm works?\n![Random Forest](https://www.ibm.com/content/dam/connectedassets-adobe-cms/worldwide-content/cdp/cf/ul/g/50/f9/ICLH_Diagram_Batch_03_27-RandomForest.component.simple-narrative-xl.ts=1712080356878.png/content/adobe-cms/us/en/topics/random-forest/jcr:content/root/table_of_contents/body/content_section_styled/content-section-body/simple_narrative0/image)\n1. **Bootstrap Sampling**: The algorithm starts by creating multiple bootstrap samples from the original training data. Each bootstrap sample is created by randomly selecting examples from the training data with replacement, resulting in some examples being duplicated and others being left out.\n2. **Decision Tree Construction**: For each bootstrap sample, a decision tree is constructed. This is typically done using a recursive partitioning algorithm, such as the CART (Classification and Regression Trees) algorithm.\n3. **Random Feature Selection**: At each node of the decision tree, a random subset of features is selected from the total set of features. This is a key aspect of random forests, as it introduces randomness and reduces the correlation between individual trees, ultimately improving the overall performance of the ensemble.\n4. **Node Splitting**: At each node of the decision tree, the algorithm selects the best feature and the best split point from the random subset of features to split the node. This process continues recursively until a stopping criterion is met, such as a maximum tree depth or a minimum number of samples in a leaf node.\n5. **Growing the Forest**: Steps 2-4 are repeated for each bootstrap sample, resulting in an ensemble of decision trees, collectively forming the random forest.\n6. **Prediction**: To make a prediction for a new data point, the point is passed through each decision tree in the forest. Each tree casts a \"vote\" for the predicted class or value, and the final prediction is determined by aggregating these votes. For classification tasks, the majority vote across all trees is taken as the predicted class. For regression tasks, the average of the predictions from all trees is used as the final prediction.\n7. **Out-of-Bag (OOB) Error Estimation**: During the construction of each tree, the observations that are not included in the bootstrap sample (the out-of-bag observations) can be used to estimate the generalization error of that tree. The OOB error is then averaged across all trees to provide an unbiased estimate of the random forest's generalization error.\n8. **Feature Importance**: Random forests can also provide a measure of the importance of each feature in the dataset. This is typically done by evaluating how much the prediction error increases when the values of a particular feature are randomly permuted.","metadata":{}},{"cell_type":"code","source":"class RandomForest:\n    \"\"\"\n    Random Forest Classifier\n\n    Parameters\n    ----------\n    n_estimators : int, default = 100\n        The number of trees in the random forest.\n    max_depth : int, default = None\n        The maximum depth of each decision tree in the random forest.\n        If None, the tree will expand until all leaves contain less than min_samples_split samples.\n    min_samples_split : int, default = 2\n        The minimum number of samples required to split an internal node.\n    min_samples_leaf : int, default = 1\n        The minimum number of samples required to be at a leaf node.\n    max_features : {'auto', 'sqrt', 'log2'} or int, default='auto'\n        The number of features to consider when looking for the best split.\n        - 'auto' uses `sqrt(n_features)`.\n        - 'sqrt' uses `sqrt(n_features)`.\n        - 'log2' uses `log2(n_features)`.\n        - If int, use `max_features` as the maximum number of features.\n    bootstrap : bool, default=True\n        Whether bootstrap samples are used when building trees.\n    oob_score : bool, default=True\n        Whether to use out-of-bag samples to estimate the generalization score.\n    random_state : int, default=None\n        The seed used by the random number generator.\n\n    Attributes\n    ----------\n    n_estimators : int\n    max_depth : int\n    min_samples_split : int\n    min_samples_leaf : int\n    max_features : {'auto', 'sqrt', 'log2'} or int\n    bootstrap : bool\n    oob_score : bool\n    random_state : int\n    trees: list of DecisionTreeClassifier\n    criterion (str): The function to measure the quality of a split. Default is 'gini'.\n    \"\"\"\n\n    def __init__(self, n_estimators=100, max_depth=None, min_samples_split=2, min_samples_leaf=1,\n                 max_features='auto', bootstrap=True, oob_score=True, random_state=None, criterion='gini'):\n        self.n_estimators = n_estimators\n        self.max_depth = max_depth\n        self.min_samples_split = min_samples_split\n        self.min_samples_leaf = min_samples_leaf\n        self.max_features = max_features\n        self.bootstrap = bootstrap\n        self.oob_score = oob_score\n        self.random_state = random_state\n        self.trees = []\n        self.criterion = criterion\n\n    def fit(self, X, y):\n        \"\"\"\n        Build a random forest classifier from the training set (X, y).\n\n        Parameters\n        ----------\n        X : array of shape (n_samples, n_features)\n            The training input samples.\n        y : array of shape (n_samples, )\n            The target values.\n\n        Returns\n        -------\n        self : object\n            Returns self.\n        \"\"\"\n        # Create an empty list to store the trees.\n        self.trees = []\n\n        # Concatenate X and y into a single dataset.\n        dataset = np.concatenate((X, y.reshape(-1, 1)), axis=1)\n\n        # Loop over the number of trees.\n        for _ in range(self.n_estimators):\n            # Create a decision tree instance.\n            tree = DecisionTreeClassifier(max_depth=self.max_depth, min_samples_split=self.min_samples_split,\n                                          min_samples_leaf=self.min_samples_leaf, max_features=self.max_features,\n                                          criterion=self.criterion, random_state=self.random_state)\n            # Sample from the dataset with replacement (bootstrapping).\n            dataset_sample = self.bootstrap_samples(dataset)\n            # Get the X and y samples from the dataset sample.\n            X_sample, y_sample = dataset_sample[:, :-1], dataset_sample[:, -1]\n            # Fit the tree to the X and y samples.\n            tree.fit(X_sample, y_sample)\n            # Store the tree in the list of trees.\n            self.trees.append(tree)\n\n        if self.oob_score:\n            self.oob_score_ = self.calculate_oob_score(X, y)\n\n        return self\n\n    def bootstrap_samples(self, dataset):\n        \"\"\"\n        Bootstrap the dataset by sampling from it with replacement.\n\n        Parameters\n        ----------\n        dataset : array of shape (n_samples, n_features + 1(target))\n            The dataset to bootstrap\n\n        Returns\n        -------\n        dataset_sample : array of shape (n_samples, n_features + 1(target))\n            The bootstrapped dataset sample.\n        \"\"\"\n        # Get the number of samples in the dataset.\n        n_samples = dataset.shape[0]\n        # Generate random indices to index into the dataset with replacement.\n        np.random.seed(self.random_state)\n        indices = np.random.choice(n_samples, n_samples, replace=True)\n        # Return the bootstrapped dataset sample using the generated indices.\n        dataset_sample = dataset[indices]\n        return dataset_sample\n\n    def calculate_oob_score(self, X, y):\n        \"\"\"\n        Calculate the out-of-bag (OOB) score.\n\n        Parameters\n        ----------\n        X : array of shape (n_samples, n_features)\n            The input samples.\n        y : array of shape (n_samples, )\n            The target values.\n\n        Returns\n        -------\n        oob_score : float\n            The out-of-bag score.\n        \"\"\"\n        oob_predictions = np.zeros_like(y, dtype=y.dtype)\n        n_samples = X.shape[0]\n\n        for i in range(self.n_estimators):\n            tree = self.trees[i]\n            unsampled_indices = [j for j in range(n_samples) if j not in self.bootstrap_samples(np.arange(n_samples))]\n            oob_predictions[unsampled_indices] += tree.predict(X[unsampled_indices]).astype(y.dtype)\n\n        oob_predictions = np.round(oob_predictions / self.n_estimators).astype(y.dtype)\n        oob_score = (y == oob_predictions).mean()\n        return oob_score\n    \n    def predict(self, X):\n        \"\"\"\n        Predict class for X.\n\n        Parameters\n        ----------\n        X : array of shape (n_samples, n_features)\n            The input samples\n\n        Returns\n        -------\n        majority_predictions : array of shape (n_samples)\n            The predicted classes.\n        \"\"\"\n        # Get prediction from each tree in the tree list on the test data\n        predictions = np.array([tree.predict(X) for tree in self.trees])\n        # Get prediction for the same sample from all trees for each sample in the test data\n        preds = np.swapaxes(predictions, 0, 1)\n        # Get the most voted value by the trees and store it in the final predictions array\n        majority_predictions = np.array([self.most_common_label(pred) for pred in preds])\n\n        return majority_predictions\n\n    def most_common_label(self, y):\n        \"\"\"\n        Returns the most common label in an array of labels.\n\n        Parameters\n        ----------\n        y : array of shape (n_samples, )\n             The array of labels(target).\n\n        Returns\n        -------\n        most_occuring_value : int or float\n            The most common label in the array.\n        \"\"\"\n        y = list(y)\n        # Get the highest present class in the array\n        most_occuring_value = max(y, key=y.count)\n        return most_occuring_value","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:08.246160Z","iopub.execute_input":"2024-04-22T08:54:08.247041Z","iopub.status.idle":"2024-04-22T08:54:08.268538Z","shell.execute_reply.started":"2024-04-22T08:54:08.246998Z","shell.execute_reply":"2024-04-22T08:54:08.266925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RandomForest(min_samples_split=5, n_estimators=150)\nmodel.fit(X_train, y_train)\n\npredictions = model.predict(X_test) #evaluate the model on the test data","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:54:08.845085Z","iopub.execute_input":"2024-04-22T08:54:08.845583Z","iopub.status.idle":"2024-04-22T08:55:06.569504Z","shell.execute_reply.started":"2024-04-22T08:54:08.845546Z","shell.execute_reply":"2024-04-22T08:55:06.568045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Model Evaluation","metadata":{}},{"cell_type":"code","source":"def accuracy(y_true, y_pred):\n    \"\"\"\n    Computes the accuracy of a classification model.\n\n    Parameters:\n    y_true (numpy array): A numpy array of true labels for each data point.\n    y_pred (numpy array): A numpy array of predicted labels for each data point.\n\n    Returns:\n    float: The accuracy of the model, expressed as a percentage.\n    \"\"\"\n    y_true = y_true.flatten()\n    total_samples = len(y_true)\n    correct_predictions = np.sum(y_true == y_pred)\n    return (correct_predictions / total_samples)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:55:06.571697Z","iopub.execute_input":"2024-04-22T08:55:06.572078Z","iopub.status.idle":"2024-04-22T08:55:06.578130Z","shell.execute_reply.started":"2024-04-22T08:55:06.572045Z","shell.execute_reply":"2024-04-22T08:55:06.577214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def confusion_matrix(y_true, y_pred):\n        \"\"\"\n        Computes the confusion matrix.\n\n        Parameters:\n            y_true (array-like): True labels.\n            y_pred (array-like): Predicted labels.\n\n        Returns:\n            ndarray: Confusion matrix.\n        \"\"\"\n        classes = np.unique(np.concatenate((y_true, y_pred)))\n        num_classes = len(classes)\n        cm = np.zeros((num_classes, num_classes), dtype=int)\n        for i in range(len(y_true)):\n            true_label_index = np.where(classes == y_true[i])[0][0]\n            pred_label_index = np.where(classes == y_pred[i])[0][0]\n            cm[true_label_index][pred_label_index] += 1\n        return cm","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:55:06.579468Z","iopub.execute_input":"2024-04-22T08:55:06.580519Z","iopub.status.idle":"2024-04-22T08:55:06.594320Z","shell.execute_reply.started":"2024-04-22T08:55:06.580483Z","shell.execute_reply":"2024-04-22T08:55:06.593128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def classification_report(y_true, y_pred):\n        \"\"\"\n        Computes the classification report.\n\n        Parameters:\n            y_true (array-like): True labels.\n            y_pred (array-like): Predicted labels.\n\n        Returns:\n            str: Classification report.\n        \"\"\"\n        cm = confusion_matrix(y_true, y_pred)\n        classes = np.unique(np.concatenate((y_true, y_pred)))\n        num_classes = len(classes)\n\n        report = \"              precision    recall  f1-score   support\\n\\n\"\n        for i in range(num_classes):\n            precision = cm[i][i] / sum(cm[:, i]) if sum(cm[:, i]) > 0 else 0\n            recall = cm[i][i] / sum(cm[i, :]) if sum(cm[i, :]) > 0 else 0\n            f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n            support = sum(cm[i, :])\n            report += f\"Class {classes[i]}     {precision:.2f}       {recall:.2f}      {f1_score:.2f}       {support}\\n\"\n\n        return report","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:55:06.596509Z","iopub.execute_input":"2024-04-22T08:55:06.597253Z","iopub.status.idle":"2024-04-22T08:55:06.613409Z","shell.execute_reply.started":"2024-04-22T08:55:06.597216Z","shell.execute_reply":"2024-04-22T08:55:06.612255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_conf_matrix = confusion_matrix(y_test, predictions)\nrf_acc_score = accuracy(y_test, predictions)\nacc = accuracy(y_test, predictions)\n\nprint(\"Accuracy of the Random Forest is:\", acc * 100, \"%\")\nprint(rf_conf_matrix)\nprint(classification_report(y_test, predictions))","metadata":{"execution":{"iopub.status.busy":"2024-04-22T08:55:06.615310Z","iopub.execute_input":"2024-04-22T08:55:06.615746Z","iopub.status.idle":"2024-04-22T08:55:06.636983Z","shell.execute_reply.started":"2024-04-22T08:55:06.615710Z","shell.execute_reply":"2024-04-22T08:55:06.635800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that the accuracy comes out to be around 87%!! Even the f1-score is good!","metadata":{}}]}