Disclaimer: this is my first competition and I am not an expert.
\n\nI made a piece of code to convert co-occurrence matrix of cards vs merchants (id's, groups) and cards vs city. The code is able to deal with dimension problem (matrix 300k x 300k). All first level features calculated by the algorithm jump up in the list of features sorted by descending gain and split importance (LibGBM). Not to the top, but high to the middle. \nBut the problem is that the CV is not improving.
\n\nPlease give an advise where I am wrong?
\n\nThe code below assumes that card_id and merchant_id are converted to integers.
\n\n```\n hist_coocurrance_matrix.columns=cards[[\"card_id\"]].copy()\n new_coocurrance_matrix==cards[[\"card_id\"]].copy()
\n\nprint(\"CoOcurance Matrix with Dimension reductions\")\n# convert pair of card_id and second feature (merchant_id, sector_id, city_id etc) into a coocurance matrix. Then reduce dimensions\nfrom scipy.sparse import lil_matrix, dok_matrix\nfrom sklearn.decomposition import TruncatedSVD # https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.TruncatedSVD.html#sklearn.decomposition.TruncatedSVD\n\ndef feature_factorization(df_in, second_feature,col_prefix, n_out_dimensions):\n print(\"Filling co-occurance for \", second_feature)\n features = [\"card_id\", second_feature]\n f_size = [int(df_in[f].max()) + 1 for f in features]\n print(\"expected hypotetical sparce matrix size\", f_size)\n X = np.log(df_in.groupby(features)[\"purchase_amount\"].count())+1\n XX=pd.DataFrame(X, dtype=np.float32)\n XX.reset_index(inplace=True)\n del(X)\n\n # create empty sparce matrix with a type that allows random access\n co_occurrence_matrix = dok_matrix((f_size[0], f_size[1]), dtype=np.float32)\n\n\n for ix, vl in XX.iterrows(): # fill in the sparse matrix value by value\n co_occurrence_matrix[vl[\"card_id\"],vl[second_feature]]=vl[\"purchase_amount\"] # actually - count\n if int(ix/100000)*100000==ix:\n print(ix)\n\n\n svd = TruncatedSVD(n_components=n_out_dimensions, n_iter=7, random_state=42) # TO DO try algorithm=“arpack” \n svd.fit(co_occurrence_matrix)\n\n out=svd.transform(co_occurrence_matrix)\n\n print(\"svd.explained_variance_ratio_\", svd.explained_variance_ratio_) #The variance of the training samples transformed by a projection to each component.\n print(\"svd.explained_variance_ratio_.sum()\",svd.explained_variance_ratio_.sum()) # Percentage of variance explained by each of the selected components.\n print(\"svd.singular_values\", svd.singular_values_) # The singular values corresponding to each of the selected components. The singular values are equal to the 2-norms of the n_components variables in the lower-dimensional space.\n\n\n\n ou=pd.DataFrame(out, dtype=np.float32)\n ou.columns = [col_prefix + \"_\" + str(c) if c != \"card_id\" else c for c in ou.columns]\n\n #extract card_id from index\n ou.reset_index(inplace=True)\n ou.rename(columns={\"index\": \"card_id\"}, inplace=True)\n\n return ou\n\nfor col in [\"merchant_id\", \"city_id\",\"merchant_group_id\"]:\n print(f\"{time.time() - t0:.1f} seconds\")\n print(\"Cooccurance for \",col)\n t0 = time.time()\n hist_coocurrance_matrix=pd.merge(hist_coocurrance_matrix,feature_factorization(historical_transactions, col,\"hist_coocrnc_\"+col+\"_\", 10), on=\"card_id\", how=\"inner\")\n print(f\"{time.time() - t0:.1f} seconds\")\n t0 = time.time()\n new_coocurrance_matrix=pd.merge(new_coocurrance_matrix,feature_factorization(new_transactions, col,\"new_coocrnc_\"+col+\"_\", 10), on=\"card_id\", how=\"inner\")\n print(f\"{time.time() - t0:.1f} seconds\")\n\n\n# hist_coocurrance_matrix is merged to train and test\n# new_coocurrance_matrix is merged to train and test\n```
", "messages": [ { "id": 461344, "postDate": "2019-01-25T19:50:02.173Z", "content": "Disclaimer: this is my first competition and I am not an expert.
\n\nI made a piece of code to convert co-occurrence matrix of cards vs merchants (id's, groups) and cards vs city. The code is able to deal with dimension problem (matrix 300k x 300k). All first level features calculated by the algorithm jump up in the list of features sorted by descending gain and split importance (LibGBM). Not to the top, but high to the middle. \nBut the problem is that the CV is not improving.
\n\nPlease give an advise where I am wrong?
\n\nThe code below assumes that card_id and merchant_id are converted to integers.
\n\n```\n hist_coocurrance_matrix.columns=cards[[\"card_id\"]].copy()\n new_coocurrance_matrix==cards[[\"card_id\"]].copy()
\n\nprint(\"CoOcurance Matrix with Dimension reductions\")\n# convert pair of card_id and second feature (merchant_id, sector_id, city_id etc) into a coocurance matrix. Then reduce dimensions\nfrom scipy.sparse import lil_matrix, dok_matrix\nfrom sklearn.decomposition import TruncatedSVD # https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.TruncatedSVD.html#sklearn.decomposition.TruncatedSVD\n\ndef feature_factorization(df_in, second_feature,col_prefix, n_out_dimensions):\n print(\"Filling co-occurance for \", second_feature)\n features = [\"card_id\", second_feature]\n f_size = [int(df_in[f].max()) + 1 for f in features]\n print(\"expected hypotetical sparce matrix size\", f_size)\n X = np.log(df_in.groupby(features)[\"purchase_amount\"].count())+1\n XX=pd.DataFrame(X, dtype=np.float32)\n XX.reset_index(inplace=True)\n del(X)\n\n # create empty sparce matrix with a type that allows random access\n co_occurrence_matrix = dok_matrix((f_size[0], f_size[1]), dtype=np.float32)\n\n\n for ix, vl in XX.iterrows(): # fill in the sparse matrix value by value\n co_occurrence_matrix[vl[\"card_id\"],vl[second_feature]]=vl[\"purchase_amount\"] # actually - count\n if int(ix/100000)*100000==ix:\n print(ix)\n\n\n svd = TruncatedSVD(n_components=n_out_dimensions, n_iter=7, random_state=42) # TO DO try algorithm=“arpack” \n svd.fit(co_occurrence_matrix)\n\n out=svd.transform(co_occurrence_matrix)\n\n print(\"svd.explained_variance_ratio_\", svd.explained_variance_ratio_) #The variance of the training samples transformed by a projection to each component.\n print(\"svd.explained_variance_ratio_.sum()\",svd.explained_variance_ratio_.sum()) # Percentage of variance explained by each of the selected components.\n print(\"svd.singular_values\", svd.singular_values_) # The singular values corresponding to each of the selected components. The singular values are equal to the 2-norms of the n_components variables in the lower-dimensional space.\n\n\n\n ou=pd.DataFrame(out, dtype=np.float32)\n ou.columns = [col_prefix + \"_\" + str(c) if c != \"card_id\" else c for c in ou.columns]\n\n #extract card_id from index\n ou.reset_index(inplace=True)\n ou.rename(columns={\"index\": \"card_id\"}, inplace=True)\n\n return ou\n\nfor col in [\"merchant_id\", \"city_id\",\"merchant_group_id\"]:\n print(f\"{time.time() - t0:.1f} seconds\")\n print(\"Cooccurance for \",col)\n t0 = time.time()\n hist_coocurrance_matrix=pd.merge(hist_coocurrance_matrix,feature_factorization(historical_transactions, col,\"hist_coocrnc_\"+col+\"_\", 10), on=\"card_id\", how=\"inner\")\n print(f\"{time.time() - t0:.1f} seconds\")\n t0 = time.time()\n new_coocurrance_matrix=pd.merge(new_coocurrance_matrix,feature_factorization(new_transactions, col,\"new_coocrnc_\"+col+\"_\", 10), on=\"card_id\", how=\"inner\")\n print(f\"{time.time() - t0:.1f} seconds\")\n\n\n# hist_coocurrance_matrix is merged to train and test\n# new_coocurrance_matrix is merged to train and test\n```
", "rawMarkdown": "Disclaimer: this is my first competition and I am not an expert.\n\nI made a piece of code to convert co-occurrence matrix of cards vs merchants (id's, groups) and cards vs city. The code is able to deal with dimension problem (matrix 300k x 300k). All first level features calculated by the algorithm jump up in the list of features sorted by descending gain and split importance (LibGBM). Not to the top, but high to the middle. \nBut the problem is that the CV is not improving. \n\nPlease give an advise where I am wrong?\n\n\n\nThe code below assumes that card_id and merchant_id are converted to integers.\n\n```\n hist_coocurrance_matrix.columns=cards[[\"card_id\"]].copy()\n new_coocurrance_matrix==cards[[\"card_id\"]].copy()\n\n\n print(\"CoOcurance Matrix with Dimension reductions\")\n # convert pair of card_id and second feature (merchant_id, sector_id, city_id etc) into a coocurance matrix. Then reduce dimensions\n from scipy.sparse import lil_matrix, dok_matrix\n from sklearn.decomposition import TruncatedSVD # https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.TruncatedSVD.html#sklearn.decomposition.TruncatedSVD\n\n def feature_factorization(df_in, second_feature,col_prefix, n_out_dimensions):\n print(\"Filling co-occurance for \", second_feature)\n features = [\"card_id\", second_feature]\n f_size = [int(df_in[f].max()) + 1 for f in features]\n print(\"expected hypotetical sparce matrix size\", f_size)\n X = np.log(df_in.groupby(features)[\"purchase_amount\"].count())+1\n XX=pd.DataFrame(X, dtype=np.float32)\n XX.reset_index(inplace=True)\n del(X)\n\n # create empty sparce matrix with a type that allows random access\n co_occurrence_matrix = dok_matrix((f_size[0], f_size[1]), dtype=np.float32)\n\n\n for ix, vl in XX.iterrows(): # fill in the sparse matrix value by value\n co_occurrence_matrix[vl[\"card_id\"],vl[second_feature]]=vl[\"purchase_amount\"] # actually - count\n if int(ix/100000)*100000==ix:\n print(ix)\n\n \n svd = TruncatedSVD(n_components=n_out_dimensions, n_iter=7, random_state=42) # TO DO try algorithm=“arpack” \n svd.fit(co_occurrence_matrix)\n\n out=svd.transform(co_occurrence_matrix)\n\n print(\"svd.explained_variance_ratio_\", svd.explained_variance_ratio_) #The variance of the training samples transformed by a projection to each component.\n print(\"svd.explained_variance_ratio_.sum()\",svd.explained_variance_ratio_.sum()) # Percentage of variance explained by each of the selected components.\n print(\"svd.singular_values\", svd.singular_values_) # The singular values corresponding to each of the selected components. The singular values are equal to the 2-norms of the n_components variables in the lower-dimensional space.\n\n\n \n ou=pd.DataFrame(out, dtype=np.float32)\n ou.columns = [col_prefix + \"_\" + str(c) if c != \"card_id\" else c for c in ou.columns]\n \n #extract card_id from index\n ou.reset_index(inplace=True)\n ou.rename(columns={\"index\": \"card_id\"}, inplace=True)\n \n return ou\n\n for col in [\"merchant_id\", \"city_id\",\"merchant_group_id\"]:\n print(f\"{time.time() - t0:.1f} seconds\")\n print(\"Cooccurance for \",col)\n t0 = time.time()\n hist_coocurrance_matrix=pd.merge(hist_coocurrance_matrix,feature_factorization(historical_transactions, col,\"hist_coocrnc_\"+col+\"_\", 10), on=\"card_id\", how=\"inner\")\n print(f\"{time.time() - t0:.1f} seconds\")\n t0 = time.time()\n new_coocurrance_matrix=pd.merge(new_coocurrance_matrix,feature_factorization(new_transactions, col,\"new_coocrnc_\"+col+\"_\", 10), on=\"card_id\", how=\"inner\")\n print(f\"{time.time() - t0:.1f} seconds\")\n \n\n # hist_coocurrance_matrix is merged to train and test\n # new_coocurrance_matrix is merged to train and test\n```\n", "votes": 1 } ], "comments": [], "raw_markdown_by_id": { "461344": "Disclaimer: this is my first competition and I am not an expert.\n\nI made a piece of code to convert co-occurrence matrix of cards vs merchants (id's, groups) and cards vs city. The code is able to deal with dimension problem (matrix 300k x 300k). All first level features calculated by the algorithm jump up in the list of features sorted by descending gain and split importance (LibGBM). Not to the top, but high to the middle. \nBut the problem is that the CV is not improving. \n\nPlease give an advise where I am wrong?\n\n\n\nThe code below assumes that card_id and merchant_id are converted to integers.\n\n```\n hist_coocurrance_matrix.columns=cards[[\"card_id\"]].copy()\n new_coocurrance_matrix==cards[[\"card_id\"]].copy()\n\n\n print(\"CoOcurance Matrix with Dimension reductions\")\n # convert pair of card_id and second feature (merchant_id, sector_id, city_id etc) into a coocurance matrix. Then reduce dimensions\n from scipy.sparse import lil_matrix, dok_matrix\n from sklearn.decomposition import TruncatedSVD # https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.TruncatedSVD.html#sklearn.decomposition.TruncatedSVD\n\n def feature_factorization(df_in, second_feature,col_prefix, n_out_dimensions):\n print(\"Filling co-occurance for \", second_feature)\n features = [\"card_id\", second_feature]\n f_size = [int(df_in[f].max()) + 1 for f in features]\n print(\"expected hypotetical sparce matrix size\", f_size)\n X = np.log(df_in.groupby(features)[\"purchase_amount\"].count())+1\n XX=pd.DataFrame(X, dtype=np.float32)\n XX.reset_index(inplace=True)\n del(X)\n\n # create empty sparce matrix with a type that allows random access\n co_occurrence_matrix = dok_matrix((f_size[0], f_size[1]), dtype=np.float32)\n\n\n for ix, vl in XX.iterrows(): # fill in the sparse matrix value by value\n co_occurrence_matrix[vl[\"card_id\"],vl[second_feature]]=vl[\"purchase_amount\"] # actually - count\n if int(ix/100000)*100000==ix:\n print(ix)\n\n \n svd = TruncatedSVD(n_components=n_out_dimensions, n_iter=7, random_state=42) # TO DO try algorithm=“arpack” \n svd.fit(co_occurrence_matrix)\n\n out=svd.transform(co_occurrence_matrix)\n\n print(\"svd.explained_variance_ratio_\", svd.explained_variance_ratio_) #The variance of the training samples transformed by a projection to each component.\n print(\"svd.explained_variance_ratio_.sum()\",svd.explained_variance_ratio_.sum()) # Percentage of variance explained by each of the selected components.\n print(\"svd.singular_values\", svd.singular_values_) # The singular values corresponding to each of the selected components. The singular values are equal to the 2-norms of the n_components variables in the lower-dimensional space.\n\n\n \n ou=pd.DataFrame(out, dtype=np.float32)\n ou.columns = [col_prefix + \"_\" + str(c) if c != \"card_id\" else c for c in ou.columns]\n \n #extract card_id from index\n ou.reset_index(inplace=True)\n ou.rename(columns={\"index\": \"card_id\"}, inplace=True)\n \n return ou\n\n for col in [\"merchant_id\", \"city_id\",\"merchant_group_id\"]:\n print(f\"{time.time() - t0:.1f} seconds\")\n print(\"Cooccurance for \",col)\n t0 = time.time()\n hist_coocurrance_matrix=pd.merge(hist_coocurrance_matrix,feature_factorization(historical_transactions, col,\"hist_coocrnc_\"+col+\"_\", 10), on=\"card_id\", how=\"inner\")\n print(f\"{time.time() - t0:.1f} seconds\")\n t0 = time.time()\n new_coocurrance_matrix=pd.merge(new_coocurrance_matrix,feature_factorization(new_transactions, col,\"new_coocrnc_\"+col+\"_\", 10), on=\"card_id\", how=\"inner\")\n print(f\"{time.time() - t0:.1f} seconds\")\n \n\n # hist_coocurrance_matrix is merged to train and test\n # new_coocurrance_matrix is merged to train and test\n```\n" } }