{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-25T09:46:17.226230Z","iopub.execute_input":"2023-03-25T09:46:17.226974Z","iopub.status.idle":"2023-03-25T09:46:17.233406Z","shell.execute_reply.started":"2023-03-25T09:46:17.226935Z","shell.execute_reply":"2023-03-25T09:46:17.232041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using tabular and Ad banner Image features to learn to predict CTR probability","metadata":{}},{"cell_type":"code","source":"print(os.listdir('/kaggle/input/avazu-ctr-prediction'))","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:46:18.882604Z","iopub.execute_input":"2023-03-25T09:46:18.883092Z","iopub.status.idle":"2023-03-25T09:46:18.891066Z","shell.execute_reply.started":"2023-03-25T09:46:18.883049Z","shell.execute_reply":"2023-03-25T09:46:18.889855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\nsns.set_theme(style=\"darkgrid\")\npd.set_option('display.max_columns', 0)\nplt.style.use('ggplot')\npd.options.display.float_format = '{:.2f}'.format\nimport math\nfrom sklearn.preprocessing import LabelEncoder \nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.dummy import DummyClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom IPython.display import Image  \nfrom six import StringIO\nfrom sklearn.tree import export_graphviz\nimport graphviz\nfrom sklearn.ensemble import RandomForestClassifier\nimport statsmodels.api as sm\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom sklearn.feature_selection import RFE\nfrom sklearn.model_selection import GridSearchCV\nimport random\nimport gzip\nimport category_encoders as ce\npd.options.display.float_format = '{:.2f}'.format","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:46:19.278343Z","iopub.execute_input":"2023-03-25T09:46:19.279087Z","iopub.status.idle":"2023-03-25T09:46:19.288481Z","shell.execute_reply.started":"2023-03-25T09:46:19.279049Z","shell.execute_reply":"2023-03-25T09:46:19.287410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading datasets","metadata":{}},{"cell_type":"code","source":"# Reading the input data\n\nnum_records = 40428967\nsample_size = 5000000\nskip_values = sorted(random.sample(range(1,num_records), num_records - sample_size))\nparse_date = lambda val : pd.datetime.strptime(val, '%y%m%d%H')\n\ntrain = pd.read_csv(\"../input/avazu-ctr-prediction/train.gz\", parse_dates=['hour'], date_parser=parse_date,\\\n                     skiprows=skip_values, nrows=100000)\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:46:20.166426Z","iopub.execute_input":"2023-03-25T09:46:20.168842Z","iopub.status.idle":"2023-03-25T09:47:35.510162Z","shell.execute_reply.started":"2023-03-25T09:46:20.168792Z","shell.execute_reply":"2023-03-25T09:47:35.508895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading the test data.\ndef f_read_test_data():\n    test = pd.read_csv('../input/avazu-ctr-prediction/test.gz', parse_dates=['hour'], date_parser=parse_date, nrows=10000)\n    return test.copy()\ntest = f_read_test_data()\ntest.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:35.512638Z","iopub.execute_input":"2023-03-25T09:47:35.513030Z","iopub.status.idle":"2023-03-25T09:47:35.730835Z","shell.execute_reply.started":"2023-03-25T09:47:35.512989Z","shell.execute_reply":"2023-03-25T09:47:35.729576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shape of the data read. \n\nprint('Train dataset:',train.shape)\nprint('Test dataset:',test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:35.732458Z","iopub.execute_input":"2023-03-25T09:47:35.733653Z","iopub.status.idle":"2023-03-25T09:47:35.740568Z","shell.execute_reply.started":"2023-03-25T09:47:35.733611Z","shell.execute_reply":"2023-03-25T09:47:35.739275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:35.743593Z","iopub.execute_input":"2023-03-25T09:47:35.744941Z","iopub.status.idle":"2023-03-25T09:47:35.799455Z","shell.execute_reply.started":"2023-03-25T09:47:35.744900Z","shell.execute_reply":"2023-03-25T09:47:35.798147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hour column contains event date with all the details, extracting the same to create different feature columns.\n\ntrain['month'] = train['hour'].dt.month\ntrain['dayofweek'] = train['hour'].dt.dayofweek\ntrain['day'] = train['hour'].dt.day\ntrain['hour_time'] = train['hour'].dt.hour\ntrain.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:35.801699Z","iopub.execute_input":"2023-03-25T09:47:35.802166Z","iopub.status.idle":"2023-03-25T09:47:35.881430Z","shell.execute_reply.started":"2023-03-25T09:47:35.802122Z","shell.execute_reply":"2023-03-25T09:47:35.880214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore types of Features","metadata":{}},{"cell_type":"code","source":"#Seggregating numerical and categorical variables using data type.\n\nnumerical = []\ncategorical = []\n\nfor col in (train.columns):\n    if train[col].dtype == \"object\":\n        categorical.append(col)\n    else:\n        numerical.append(col)\nprint(\"numerical columns = \",numerical)\nprint(\"\\ncategorical columns = \",categorical)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:35.883509Z","iopub.execute_input":"2023-03-25T09:47:35.883938Z","iopub.status.idle":"2023-03-25T09:47:35.894683Z","shell.execute_reply.started":"2023-03-25T09:47:35.883887Z","shell.execute_reply":"2023-03-25T09:47:35.893341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analyse Numerical Columns","metadata":{}},{"cell_type":"code","source":"train[numerical].hist(figsize=(20,14))","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:35.896170Z","iopub.execute_input":"2023-03-25T09:47:35.897431Z","iopub.status.idle":"2023-03-25T09:47:40.183746Z","shell.execute_reply.started":"2023-03-25T09:47:35.897350Z","shell.execute_reply":"2023-03-25T09:47:40.182696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Looks like there are outliers in C15,C16, C19","metadata":{}},{"cell_type":"code","source":"# Cap the outliers to remove records where column values are above 98 percentile\nimport numpy as np\nOutlier_Col = ['C15', 'C16', 'C19']\nfor col in Outlier_Col:\n    percentiles = train[col].quantile(0.98)\n    if train[col].quantile(0.98) < 0.5 * train[col].max():\n        train[col][train[col] >= percentiles] = percentiles\n        print( col, np.percentile(train[col],98),train[col].quantile(0.98) )","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:40.185279Z","iopub.execute_input":"2023-03-25T09:47:40.186567Z","iopub.status.idle":"2023-03-25T09:47:40.218506Z","shell.execute_reply.started":"2023-03-25T09:47:40.186504Z","shell.execute_reply":"2023-03-25T09:47:40.217179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## What are the redundant features?","metadata":{}},{"cell_type":"code","source":"# Pearson correlation table to find the relationship with output of all input features. \n\ncorr = train.corr()\nf, ax = plt.subplots(figsize=(10,10))\nsns.heatmap(corr, ax=ax, annot=True,linewidths=3,cmap='YlGn')\nplt.title(\"Pearson correlation of Features\", y=1.05, size=15)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:40.220405Z","iopub.execute_input":"2023-03-25T09:47:40.221277Z","iopub.status.idle":"2023-03-25T09:47:41.744414Z","shell.execute_reply.started":"2023-03-25T09:47:40.221236Z","shell.execute_reply":"2023-03-25T09:47:41.743367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['C20'].value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:41.748951Z","iopub.execute_input":"2023-03-25T09:47:41.750104Z","iopub.status.idle":"2023-03-25T09:47:41.762994Z","shell.execute_reply.started":"2023-03-25T09:47:41.750066Z","shell.execute_reply":"2023-03-25T09:47:41.761359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Drop unwanted columns\n### 1. Month has a constant value and C20 has large percentage of -1 values so drop them\n### 2. C14, C17 are highly correlated drop either of them","metadata":{}},{"cell_type":"code","source":"# as said above dropping columns. \n\ntrain.drop(['month', 'C20','C14'], axis=1, inplace=True)\ntrain.columns","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:41.764469Z","iopub.execute_input":"2023-03-25T09:47:41.765767Z","iopub.status.idle":"2023-03-25T09:47:41.786173Z","shell.execute_reply.started":"2023-03-25T09:47:41.765727Z","shell.execute_reply":"2023-03-25T09:47:41.784971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# id column have a unique columns so nothing to learn from this column\n# hour column have been derived into different columns, hence dropping. \n# rename click to y (output)\n# after dropping hour column, hour_time can be made as hour\n\ntrain.drop(['id', 'hour'], axis = 1, inplace = True) \ntrain.rename(columns={'click': 'y',\n                   'hour_time': 'hour'},\n          inplace=True, errors='raise')\n\ntrain.columns","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:41.788092Z","iopub.execute_input":"2023-03-25T09:47:41.788506Z","iopub.status.idle":"2023-03-25T09:47:41.812743Z","shell.execute_reply.started":"2023-03-25T09:47:41.788469Z","shell.execute_reply":"2023-03-25T09:47:41.811808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analyse Categorical/ Sparse Features","metadata":{}},{"cell_type":"code","source":"train.y.value_counts().plot(kind='bar')\nprint(train.y.value_counts())","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:41.814425Z","iopub.execute_input":"2023-03-25T09:47:41.814782Z","iopub.status.idle":"2023-03-25T09:47:42.031770Z","shell.execute_reply.started":"2023-03-25T09:47:41.814747Z","shell.execute_reply":"2023-03-25T09:47:42.030704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,4))\nsns.countplot(x='banner_pos',hue='y',data=train)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:42.033113Z","iopub.execute_input":"2023-03-25T09:47:42.034918Z","iopub.status.idle":"2023-03-25T09:47:42.335557Z","shell.execute_reply.started":"2023-03-25T09:47:42.034877Z","shell.execute_reply":"2023-03-25T09:47:42.334509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='hour',hue='y',data=train)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:42.337335Z","iopub.execute_input":"2023-03-25T09:47:42.338104Z","iopub.status.idle":"2023-03-25T09:47:42.613950Z","shell.execute_reply.started":"2023-03-25T09:47:42.338065Z","shell.execute_reply":"2023-03-25T09:47:42.612888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in categorical:\n    print('Unique Values in ',col, ' = ',train[col].nunique(), ' :',len(train))","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:42.615536Z","iopub.execute_input":"2023-03-25T09:47:42.616274Z","iopub.status.idle":"2023-03-25T09:47:42.691694Z","shell.execute_reply.started":"2023-03-25T09:47:42.616233Z","shell.execute_reply":"2023-03-25T09:47:42.690474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,6))\nsns.countplot(x='app_category',hue='y',data=train)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:42.693637Z","iopub.execute_input":"2023-03-25T09:47:42.694039Z","iopub.status.idle":"2023-03-25T09:47:43.218650Z","shell.execute_reply.started":"2023-03-25T09:47:42.693997Z","shell.execute_reply":"2023-03-25T09:47:43.217503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Most of the categorical variables have lot of unique values in it, one hot encoding is not a scalable approach as it makes the feature vector large.\n### label encoding and scaling is a better approach","metadata":{}},{"cell_type":"markdown","source":"### sparse feature\nC1 \nbanner_pos   \nsite_id\nsite_domain \nsite_category\napp_id\napp_domain\napp_category\ndevice_id\ndevice_ip\ndevice_model\ndevice_type\ndevice_conn_type\n\n### Dense feature\nC15 to c21","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:43.220478Z","iopub.execute_input":"2023-03-25T09:47:43.221267Z","iopub.status.idle":"2023-03-25T09:47:43.262414Z","shell.execute_reply.started":"2023-03-25T09:47:43.221226Z","shell.execute_reply":"2023-03-25T09:47:43.261223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sparse_features = train.iloc[::,1:14].columns.values.tolist()\ndense_features = train.iloc[::,14:14+6].columns.values.tolist()\nsparse_features, dense_features","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:43.264331Z","iopub.execute_input":"2023-03-25T09:47:43.264716Z","iopub.status.idle":"2023-03-25T09:47:43.289687Z","shell.execute_reply.started":"2023-03-25T09:47:43.264679Z","shell.execute_reply":"2023-03-25T09:47:43.288593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler,LabelEncoder\nscaler = MinMaxScaler(feature_range=(0,1))\nencoder = LabelEncoder()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:43.291422Z","iopub.execute_input":"2023-03-25T09:47:43.291846Z","iopub.status.idle":"2023-03-25T09:47:43.297765Z","shell.execute_reply.started":"2023-03-25T09:47:43.291810Z","shell.execute_reply":"2023-03-25T09:47:43.296574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a=[1,2,3,4,1]\nencoder.fit_transform(a)\na=a+[5]\nencoder.fit_transform(a)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:43.299311Z","iopub.execute_input":"2023-03-25T09:47:43.300348Z","iopub.status.idle":"2023-03-25T09:47:43.313665Z","shell.execute_reply.started":"2023-03-25T09:47:43.300297Z","shell.execute_reply":"2023-03-25T09:47:43.312502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Encode Sparse Features and Scale Dense Features","metadata":{}},{"cell_type":"code","source":"data = train.copy()\ndata=data[sparse_features+dense_features+['y']]\nfor sparse in sparse_features:\n    data[sparse] = encoder.fit_transform(data[sparse])\ndata[dense_features] = scaler.fit_transform(data[dense_features])\ndata.shape, train.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:43.315100Z","iopub.execute_input":"2023-03-25T09:47:43.315669Z","iopub.status.idle":"2023-03-25T09:47:43.766706Z","shell.execute_reply.started":"2023-03-25T09:47:43.315631Z","shell.execute_reply":"2023-03-25T09:47:43.765462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre-process Test Data","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\ntest = f_read_test_data()\ntest['month'] = test['hour'].dt.month\ntest['dayofweek'] = test['hour'].dt.dayofweek\ntest['day'] = test['hour'].dt.day\ntest['hour_time'] = test['hour'].dt.hour\n\ntest.drop(['id', 'hour'], axis = 1, inplace = True) \ntest.rename(columns={\n                   'hour_time': 'hour'},\n          inplace=True, errors='raise')\n\ntest.columns\n\ntest_data = test.copy()\ntest_data=test_data[sparse_features+dense_features]\nfor sparse in sparse_features:\n    test_data[sparse] = encoder.fit_transform(test_data[sparse])\ntest_data[dense_features] = scaler.fit_transform(test_data[dense_features])\n\ntest_data = torch.tensor(test_data.values).float()\ntest_data.shape, train.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:43.768194Z","iopub.execute_input":"2023-03-25T09:47:43.769381Z","iopub.status.idle":"2023-03-25T09:47:44.018952Z","shell.execute_reply.started":"2023-03-25T09:47:43.769342Z","shell.execute_reply":"2023-03-25T09:47:44.017594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampled_data = data.copy() #sample(frac=0.1, random_state=42)\nX = sampled_data.drop(['y'], axis=1)\ny = sampled_data['y']\ndel sampled_data","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:44.020757Z","iopub.execute_input":"2023-03-25T09:47:44.021152Z","iopub.status.idle":"2023-03-25T09:47:44.036311Z","shell.execute_reply.started":"2023-03-25T09:47:44.021114Z","shell.execute_reply":"2023-03-25T09:47:44.035072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:44.038327Z","iopub.execute_input":"2023-03-25T09:47:44.038769Z","iopub.status.idle":"2023-03-25T09:47:44.074246Z","shell.execute_reply.started":"2023-03-25T09:47:44.038731Z","shell.execute_reply":"2023-03-25T09:47:44.073232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:47:44.075563Z","iopub.execute_input":"2023-03-25T09:47:44.076037Z","iopub.status.idle":"2023-03-25T09:47:44.088252Z","shell.execute_reply.started":"2023-03-25T09:47:44.075998Z","shell.execute_reply":"2023-03-25T09:47:44.087126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert DataFrame to Torch Tensors","metadata":{}},{"cell_type":"code","source":"X_torch = torch.tensor(X.values)\ny_torch = torch.tensor(y.values)\nX_torch.shape, y_torch.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:52:52.946819Z","iopub.execute_input":"2023-03-25T09:52:52.947742Z","iopub.status.idle":"2023-03-25T09:52:52.973808Z","shell.execute_reply.started":"2023-03-25T09:52:52.947704Z","shell.execute_reply":"2023-03-25T09:52:52.972496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X[0:32].shape, y[0:32].float().view(-1,1).shape","metadata":{"execution":{"iopub.status.busy":"2023-03-24T08:48:56.999211Z","iopub.execute_input":"2023-03-24T08:48:56.999721Z","iopub.status.idle":"2023-03-24T08:48:57.030062Z","shell.execute_reply.started":"2023-03-24T08:48:56.999675Z","shell.execute_reply":"2023-03-24T08:48:57.028862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Architecture","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\n\nclass TransformerModel(nn.Module):\n    def __init__(self, visual_embedding_dim, tabular_feature_dim, num_heads, hidden_dim, num_layers):\n        super(TransformerModel, self).__init__()\n\n        # Visual embedding encoder\n        self.visual_encoder = nn.Linear(visual_embedding_dim, hidden_dim)\n\n        # Tabular data encoder\n        self.tabular_encoder = nn.Sequential(\n            nn.Linear(tabular_feature_dim, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, hidden_dim)\n        )\n\n        # Transformer encoder\n        self.transformer_encoder_layer = nn.TransformerEncoderLayer(hidden_dim*2, num_heads, dim_feedforward=4*hidden_dim)\n        self.transformer_encoder = nn.TransformerEncoder(self.transformer_encoder_layer, num_layers)\n\n        # Output layer\n        self.output_layer = nn.Linear(hidden_dim*2, 1)\n\n    def forward(self, visual_input, tabular_input):\n        # Visual embedding encoding\n        visual_embedding = self.visual_encoder(visual_input)\n\n        # Tabular data encoding\n        #print(tabular_input)\n        tabular_embedding = self.tabular_encoder(tabular_input)\n\n        # Concatenate visual and tabular embeddings\n        #print('before concat =>',visual_embedding.shape, visual_embedding.dtype,tabular_embedding.shape)\n        combined_embedding = torch.cat((visual_embedding, tabular_embedding), dim=1)\n\n        # Transformer encoding\n        #print('After Concatenation ',combined_embedding.shape)\n        transformer_output = self.transformer_encoder(combined_embedding)\n\n        # Output layer\n        output = self.output_layer(transformer_output)\n\n        # Sigmoid activation for click-through rate\n        output = torch.sigmoid(output)\n\n        return output\n\n# Model\nmodel = TransformerModel(visual_embedding_dim=2048, tabular_feature_dim=19, num_heads=8, hidden_dim=512, num_layers=4)\nprint(model)\noptimizer = optim.Adam(model.parameters(), lr=0.001)\ncriterion = nn.BCELoss()\n\nvisual_input = torch.randn(32, 2048).float()\ntabular_input = torch.randn(32, 32)\nlabel = torch.randint(0, 2, (32, 1)).float()\n\ntabular_input = X_torch[0:32].float()\nlabel = y_torch[0:32].float().view(-1,1)\n\nprint('Input shapes Visual and tabular')\nprint('================================')\nprint(visual_input.shape, tabular_input.shape, 'Visual Dtype =>',visual_input.dtype)\noutput = model(visual_input, tabular_input)\nprint('Output ',output.shape)\nprint('Label ',label.shape)\nloss = criterion(output, label)\n\nloss.backward()\noptimizer.step()\nprint(loss)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:53:13.637636Z","iopub.execute_input":"2023-03-25T09:53:13.638329Z","iopub.status.idle":"2023-03-25T09:53:14.688242Z","shell.execute_reply.started":"2023-03-25T09:53:13.638291Z","shell.execute_reply":"2023-03-25T09:53:14.687177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.modules","metadata":{"execution":{"iopub.status.busy":"2023-03-25T10:51:13.033973Z","iopub.execute_input":"2023-03-25T10:51:13.034714Z","iopub.status.idle":"2023-03-25T10:51:13.047929Z","shell.execute_reply.started":"2023-03-25T10:51:13.034674Z","shell.execute_reply":"2023-03-25T10:51:13.046650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tabular_input\nprint(tabular_input.shape)\nemb = nn.Embedding(32,4)\nemb(torch.tensor(X.iloc[:,1], dtype=torch.int32) ).shape #, 32x (1x4), (1x4)","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:48:39.458330Z","iopub.execute_input":"2023-03-25T09:48:39.459428Z","iopub.status.idle":"2023-03-25T09:48:39.494483Z","shell.execute_reply.started":"2023-03-25T09:48:39.459382Z","shell.execute_reply":"2023-03-25T09:48:39.493441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emb.weight","metadata":{"execution":{"iopub.status.busy":"2023-03-25T09:49:46.144620Z","iopub.execute_input":"2023-03-25T09:49:46.145323Z","iopub.status.idle":"2023-03-25T09:49:46.194509Z","shell.execute_reply.started":"2023-03-25T09:49:46.145286Z","shell.execute_reply":"2023-03-25T09:49:46.193574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hyperparameters\nbatch_size = 32 # how many independent feature vectors will we process in parallel?\nmax_iters = 100\neval_interval = 2\nlearning_rate = 1e-3\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\neval_iters = 200\ndropout = 0.0\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\n\nclass TransformerModel(nn.Module):\n    def __init__(self, visual_embedding_dim, v_hidden_dim, tabular_feature_dim, num_heads, hidden_dim, num_layers):\n        super(TransformerModel, self).__init__()\n\n        # Visual embedding encoder\n        self.visual_encoder = nn.Linear(visual_embedding_dim, v_hidden_dim)\n\n        # Tabular data encoder\n        self.tabular_encoder = nn.Sequential(\n            nn.Linear(tabular_feature_dim, hidden_dim),\n            nn.ReLU(),\n            nn.Linear(hidden_dim, hidden_dim)\n        )\n\n        # Transformer encoder\n        input_hidden_dim = hidden_dim + v_hidden_dim\n        self.transformer_encoder_layer = nn.TransformerEncoderLayer(input_hidden_dim, num_heads, dim_feedforward=4*hidden_dim)\n        self.transformer_encoder = nn.TransformerEncoder(self.transformer_encoder_layer, num_layers)\n\n        # Output layer\n        self.output_layer = nn.Linear(input_hidden_dim, 1)\n\n    def forward(self, visual_input, tabular_input):\n        # Visual embedding encoding\n        if visual_input != []:\n            visual_embedding = self.visual_encoder(visual_input)\n        else:\n            visual_embedding = []\n        # Tabular data encoding\n        #print(tabular_input)\n        tabular_embedding = self.tabular_encoder(tabular_input)\n\n        # Concatenate visual and tabular embeddings\n        #print('before concat =>',visual_embedding.shape, visual_embedding.dtype,tabular_embedding.shape)\n        if visual_embedding != []:\n            combined_embedding = torch.cat((visual_embedding, tabular_embedding), dim=1)\n        else:\n            combined_embedding = tabular_embedding\n\n        # Transformer encoding\n        #print('After Concatenation ',combined_embedding.shape)\n        transformer_output = self.transformer_encoder(combined_embedding)\n\n        # Output layer\n        #print(transformer_output.shape)\n        output = self.output_layer(transformer_output)\n\n        # Sigmoid activation for click-through rate\n        output = torch.sigmoid(output)\n\n        return output\n\n# Model\nmodel = TransformerModel(visual_embedding_dim=0, v_hidden_dim=0, tabular_feature_dim=19, num_heads=8, hidden_dim=512, num_layers=4)\n#print(model)\n\n# data loading\ndef get_batch(split):\n    # generate a small batch of data of inputs x and targets y\n    ix = torch.randint(len(data), (batch_size,)) # generate 32 random indexes of feature set\n    #print(ix)\n    x = torch.stack( [X_torch[i].float() for i in ix] )\n    target = torch.stack([y_torch[i].float() for i in ix])\n    target = target.view(-1,1)\n    #print('Function : ',x.shape, target.shape)\n    x, target = x.to(device), target.to(device)\n    return x, target\n\n@torch.no_grad()\ndef estimate_loss():\n    out = {}\n    model.eval()\n    for split in ['train', 'val']:\n        losses = torch.zeros(eval_iters)\n        for k in range(eval_iters):\n            X, Y = get_batch(split)\n            #print('Eval Function ',X.shape, Y.shape)\n            visual_input = torch.randn(32, 2048).float()\n            visual_input = visual_input.to(device)\n            visual_input = []\n            output = model(visual_input, X)\n            loss = criterion(output, Y)\n            losses[k] = loss.item()\n        out[split] = losses.mean()\n    model.train()\n    return out\nm = model.to(device)\n# print the number of parameters in the model\nprint(sum(p.numel() for p in m.parameters())/1e6, 'M parameters')\n\n# create a PyTorch optimizer\noptimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\ncriterion = nn.BCELoss()\n\nfor iter in range(max_iters):\n\n    # every once in a while evaluate the loss on train and val sets\n    if iter % eval_interval == 0 or iter == max_iters - 1:\n        losses = estimate_loss()\n        print(f\"step {iter}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}\")\n\n    # sample a batch of data\n    xb, yb = get_batch('train')\n    #print(xb.shape,yb.shape)\n    # evaluate the loss\n    visual_input = torch.randn(32, 2048).float()\n    visual_input = visual_input.to(device)\n    visual_input = []\n    output = model(visual_input, xb)\n    #print(iter,output)\n    #print(iter,yb)\n    loss = criterion(output, yb)\n    optimizer.zero_grad(set_to_none=True)\n    loss.backward()\n    optimizer.step()\n","metadata":{"execution":{"iopub.status.busy":"2023-03-25T10:47:46.314852Z","iopub.execute_input":"2023-03-25T10:47:46.315284Z","iopub.status.idle":"2023-03-25T10:49:14.062778Z","shell.execute_reply.started":"2023-03-25T10:47:46.315245Z","shell.execute_reply":"2023-03-25T10:49:14.061604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction on Test Data","metadata":{}},{"cell_type":"code","source":"test_tab_x = test_data[0:32+0].to(device)\npred_ans = model(visual_input, test_tab_x)\npred_ans","metadata":{"execution":{"iopub.status.busy":"2023-03-25T10:49:18.474930Z","iopub.execute_input":"2023-03-25T10:49:18.475300Z","iopub.status.idle":"2023-03-25T10:49:18.494626Z","shell.execute_reply.started":"2023-03-25T10:49:18.475265Z","shell.execute_reply":"2023-03-25T10:49:18.493574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# # Load Criteo dataset\n# data = pd.read_csv('path/to/criteo/dataset.txt', sep='\\t', header=None)\n\n# # Take a random subset of the dataset for faster processing\n# data = data.sample(frac=0.1)\n\n# Extract categorical and numerical features\ncategorical_cols = categorical #data.columns[:13].tolist()\nnumerical_cols = numerical #data.columns[13:].tolist()\n\n# Convert categorical features to one-hot encoding\ndata[categorical_cols] = data[categorical_cols].astype('category')\ndata = pd.get_dummies(data, columns=categorical_cols)\n\n# Normalize numerical features\nnumerical_data = data[numerical_cols]\nnormalized_data = (numerical_data - numerical_data.mean()) / numerical_data.std()\ndata[numerical_cols] = normalized_data\n\n# Calculate tabular feature dimension\ntabular_feature_dim = len(data.columns)\nprint(f\"Tabular feature dimension: {tabular_feature_dim}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-03-17T15:30:05.578693Z","iopub.status.idle":"2023-03-17T15:30:05.579197Z","shell.execute_reply.started":"2023-03-17T15:30:05.578946Z","shell.execute_reply":"2023-03-17T15:30:05.578971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Embeddings","metadata":{}},{"cell_type":"code","source":"# !pip install git+https://github.com/openai/CLIP.git\nos.listdir('/kaggle/input/256x256-images/')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T14:30:01.115696Z","iopub.execute_input":"2023-03-17T14:30:01.118242Z","iopub.status.idle":"2023-03-17T14:30:01.130576Z","shell.execute_reply.started":"2023-03-17T14:30:01.118123Z","shell.execute_reply":"2023-03-17T14:30:01.128303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport requests\nimport torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torchvision import transforms\nfrom torchvision.transforms import functional as TF\nfrom tqdm.notebook import tqdm\n\nimport sys\nsys.path.append('./CLIP')\n\nimport clip\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\nprint('Using device:', device)\nclip_model = clip.load('ViT-B/16', jit=False)[0].eval().requires_grad_(False).to(device) # pretrained on ImageNet\nfor img in img_path:\n        #img = Image.open(fetch(path)).convert('RGB')\n        print('Reading img prompt..', img)\n        img = Image.open(path+img).convert('RGB')\n        img = TF.resize(img, min(side_x, side_y, *img.size), transforms.InterpolationMode.LANCZOS)\n        batch = make_cutouts(TF.to_tensor(img).unsqueeze(0).to(device))\n        img_embed = clip_model.encode_image(normalize(batch)).float()\nprint(img_embed.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T16:32:06.642959Z","iopub.execute_input":"2023-03-16T16:32:06.643460Z","iopub.status.idle":"2023-03-16T16:32:12.082160Z","shell.execute_reply.started":"2023-03-16T16:32:06.643417Z","shell.execute_reply":"2023-03-16T16:32:12.079976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}