Hi,
\nI'm very grateful for the opportunity to learn about machine learning here on \"kaggle\" with you.
\nI'm posting this here today because I have a question I'd like to ask you.
\nI would like to ask you all for your wisdom to help me solve this problem.
(Question)
\nI wrote the code as follows.
\nThis code takes 10 minutes to complete in \"kaggle kernel\".
\nHowever, when I submitted the code, it violated the 9 hour limit and showed \"Notebook timeout\".
\nWhere in the code does the process take so long?
import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\n\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\npd.set_option(\"display.max_columns\", None)\n\n\ndef reduce_mem_usage(df):\n \"\"\" iterate through all the columns of a dataframe and modify the data type\n to reduce memory usage. \n \"\"\"\n start_mem = df.memory_usage().sum() / 1024**2\n print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n\n for col in df.columns:\n col_type = df[col].dtype\n\n if col_type != object:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == \"int\":\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n\n end_mem = df.memory_usage().sum() / 1024**2\n print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n\n return df\n\n\ndf_asset_details = pd.read_csv(r\"../input/g-research-crypto-forecasting/asset_details.csv\").sort_values(\"Asset_ID\")\ndisplay(df_asset_details)\n\n\ndef read_csv_strict(file_name=\"/kaggle/input/g-research-crypto-forecasting/train.csv\"):\n df = pd.read_csv(file_name).pipe(reduce_mem_usage)\n df[\"datetime\"] = pd.to_datetime(df[\"timestamp\"], unit=\"s\")\n df = df[df[\"datetime\"] < \"2021-06-13 00:00:00\"]\n return df\n\ndf_train = read_csv_strict()\ndisplay(df_train)\n\n\n# Two new features from the competition tutorial\ndef upper_shadow(df):\n return df[\"High\"] - np.maximum(df[\"Close\"], df[\"Open\"])\n\ndef lower_shadow(df):\n return np.minimum(df[\"Close\"], df[\"Open\"]) - df[\"Low\"]\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df,row=False):\n features = []\n keys = [\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]\n\n # df_feat = df[[\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]].copy()\n df_feat = df.copy()\n df_feat[\"Upper_Shadow\"] = upper_shadow(df_feat)\n df_feat[\"Lower_Shadow\"] = lower_shadow(df_feat)\n features += [\"Upper_Shadow\", \"Lower_Shadow\",]\n\n ## Ad dsome more feats\n df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n features += [\"Close/Open\", \"Close-Open\", \"High-Low\", \"High/Low\",]\n\n if row:\n df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n else:\n df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n df_feat[\"High/Mean\"] = df_feat[\"High\"] / df_feat[\"Mean\"]\n df_feat[\"Low/Mean\"] = df_feat[\"Low\"] / df_feat[\"Mean\"]\n df_feat[\"Volume/Count\"] = df_feat[\"Volume\"] / (df_feat[\"Count\"] + 1)\n features += [\"Mean\", \"High/Mean\", \"Low/Mean\", \"Volume/Count\",]\n\n ## possible seasonality, datetime features (unlikely to me meaningful, given very short time-frames)\n ### to do: add cyclical features for seasonality\n times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n if row:\n df_feat[\"hour\"] = times.hour # .dt\n df_feat[\"dayofweek\"] = times.dayofweek \n df_feat[\"day\"] = times.day \n else:\n df_feat[\"hour\"] = times.dt.hour # .dt\n df_feat[\"dayofweek\"] = times.dt.dayofweek \n df_feat[\"day\"] = times.dt.day \n #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True) # keep original epoch time, drop string\n\n df_feat = df_feat[keys + features]\n\n return df_feat\n\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n df = df_train[df_train[\"Asset_ID\"] == asset_id]\n\n # TODO: Try different features here!\n df_proc = get_features(df)\n df_proc[\"y\"] = df[\"Target\"]\n #df_proc = df_proc.dropna(how=\"any\")\n df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n\n X = df_proc.drop(\"y\", axis=1)\n y = df_proc[\"y\"]\n\n # TODO: Try different models here!\n model = LGBMRegressor(\n n_estimators=1500, # 1500,\n num_leaves=700,\n learning_rate=0.09,\n random_state=71,\n )\n model.fit(X, y)\n return X, y, model\n\n\n%%time\nXs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details[\"Asset_ID\"], df_asset_details[\"Asset_Name\"]):\n print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)\n Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model\n\n\n%%time\n# Check the model interface\nx = get_features(df_train.iloc[1], row=True)\ny_pred = models[0].predict([x])\ny_pred[0]\n\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n for j , row in df_test.iterrows():\n\n model = models[row[\"Asset_ID\"]]\n x_test = get_features(row, row=True)\n y_pred = model.predict([x_test])[0]\n\n df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"] = y_pred\n\n\n # Print just one sample row to get a feeling of what it looks like\n if i == 0 and j == 0:\n display(x_test)\n\n # Display the first prediction dataframe\n if i == 0:\n display(df_pred)\n\n # Send submissions\n env.predict(df_pred)\nYours sincerely.
", "messages": [ { "id": "1575000", "postDate": "11/08/2021 04:53:56", "content": "Hi,
\nI'm very grateful for the opportunity to learn about machine learning here on \"kaggle\" with you.
\nI'm posting this here today because I have a question I'd like to ask you.
\nI would like to ask you all for your wisdom to help me solve this problem.
(Question)
\nI wrote the code as follows.
\nThis code takes 10 minutes to complete in \"kaggle kernel\".
\nHowever, when I submitted the code, it violated the 9 hour limit and showed \"Notebook timeout\".
\nWhere in the code does the process take so long?
import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\n\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\npd.set_option(\"display.max_columns\", None)\n\n\ndef reduce_mem_usage(df):\n \"\"\" iterate through all the columns of a dataframe and modify the data type\n to reduce memory usage. \n \"\"\"\n start_mem = df.memory_usage().sum() / 1024**2\n print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n\n for col in df.columns:\n col_type = df[col].dtype\n\n if col_type != object:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == \"int\":\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n\n end_mem = df.memory_usage().sum() / 1024**2\n print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n\n return df\n\n\ndf_asset_details = pd.read_csv(r\"../input/g-research-crypto-forecasting/asset_details.csv\").sort_values(\"Asset_ID\")\ndisplay(df_asset_details)\n\n\ndef read_csv_strict(file_name=\"/kaggle/input/g-research-crypto-forecasting/train.csv\"):\n df = pd.read_csv(file_name).pipe(reduce_mem_usage)\n df[\"datetime\"] = pd.to_datetime(df[\"timestamp\"], unit=\"s\")\n df = df[df[\"datetime\"] < \"2021-06-13 00:00:00\"]\n return df\n\ndf_train = read_csv_strict()\ndisplay(df_train)\n\n\n# Two new features from the competition tutorial\ndef upper_shadow(df):\n return df[\"High\"] - np.maximum(df[\"Close\"], df[\"Open\"])\n\ndef lower_shadow(df):\n return np.minimum(df[\"Close\"], df[\"Open\"]) - df[\"Low\"]\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df,row=False):\n features = []\n keys = [\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]\n\n # df_feat = df[[\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]].copy()\n df_feat = df.copy()\n df_feat[\"Upper_Shadow\"] = upper_shadow(df_feat)\n df_feat[\"Lower_Shadow\"] = lower_shadow(df_feat)\n features += [\"Upper_Shadow\", \"Lower_Shadow\",]\n\n ## Ad dsome more feats\n df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n features += [\"Close/Open\", \"Close-Open\", \"High-Low\", \"High/Low\",]\n\n if row:\n df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n else:\n df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n df_feat[\"High/Mean\"] = df_feat[\"High\"] / df_feat[\"Mean\"]\n df_feat[\"Low/Mean\"] = df_feat[\"Low\"] / df_feat[\"Mean\"]\n df_feat[\"Volume/Count\"] = df_feat[\"Volume\"] / (df_feat[\"Count\"] + 1)\n features += [\"Mean\", \"High/Mean\", \"Low/Mean\", \"Volume/Count\",]\n\n ## possible seasonality, datetime features (unlikely to me meaningful, given very short time-frames)\n ### to do: add cyclical features for seasonality\n times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n if row:\n df_feat[\"hour\"] = times.hour # .dt\n df_feat[\"dayofweek\"] = times.dayofweek \n df_feat[\"day\"] = times.day \n else:\n df_feat[\"hour\"] = times.dt.hour # .dt\n df_feat[\"dayofweek\"] = times.dt.dayofweek \n df_feat[\"day\"] = times.dt.day \n #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True) # keep original epoch time, drop string\n\n df_feat = df_feat[keys + features]\n\n return df_feat\n\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n df = df_train[df_train[\"Asset_ID\"] == asset_id]\n\n # TODO: Try different features here!\n df_proc = get_features(df)\n df_proc[\"y\"] = df[\"Target\"]\n #df_proc = df_proc.dropna(how=\"any\")\n df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n\n X = df_proc.drop(\"y\", axis=1)\n y = df_proc[\"y\"]\n\n # TODO: Try different models here!\n model = LGBMRegressor(\n n_estimators=1500, # 1500,\n num_leaves=700,\n learning_rate=0.09,\n random_state=71,\n )\n model.fit(X, y)\n return X, y, model\n\n\n%%time\nXs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details[\"Asset_ID\"], df_asset_details[\"Asset_Name\"]):\n print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)\n Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model\n\n\n%%time\n# Check the model interface\nx = get_features(df_train.iloc[1], row=True)\ny_pred = models[0].predict([x])\ny_pred[0]\n\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n for j , row in df_test.iterrows():\n\n model = models[row[\"Asset_ID\"]]\n x_test = get_features(row, row=True)\n y_pred = model.predict([x_test])[0]\n\n df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"] = y_pred\n\n\n # Print just one sample row to get a feeling of what it looks like\n if i == 0 and j == 0:\n display(x_test)\n\n # Display the first prediction dataframe\n if i == 0:\n display(df_pred)\n\n # Send submissions\n env.predict(df_pred)\nYours sincerely.
", "rawMarkdown": "Hi,\nI'm very grateful for the opportunity to learn about machine learning here on \"kaggle\" with you.\nI'm posting this here today because I have a question I'd like to ask you.\nI would like to ask you all for your wisdom to help me solve this problem.\n\n---\n(Question)\nI wrote the code as follows.\nThis code takes 10 minutes to complete in \"kaggle kernel\".\nHowever, when I submitted the code, it violated the 9 hour limit and showed \"Notebook timeout\".\nWhere in the code does the process take so long?\nCan you please tell me how to submit the code without getting timed out?\n---\n\n\n\n```Python\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\n\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\npd.set_option(\"display.max_columns\", None)\n\n\ndef reduce_mem_usage(df):\n \"\"\" iterate through all the columns of a dataframe and modify the data type\n to reduce memory usage. \n \"\"\"\n start_mem = df.memory_usage().sum() / 1024**2\n print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n \n for col in df.columns:\n col_type = df[col].dtype\n \n if col_type != object:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == \"int\":\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n\n end_mem = df.memory_usage().sum() / 1024**2\n print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n \n return df\n\n\ndf_asset_details = pd.read_csv(r\"../input/g-research-crypto-forecasting/asset_details.csv\").sort_values(\"Asset_ID\")\ndisplay(df_asset_details)\n\n\ndef read_csv_strict(file_name=\"/kaggle/input/g-research-crypto-forecasting/train.csv\"):\n df = pd.read_csv(file_name).pipe(reduce_mem_usage)\n df[\"datetime\"] = pd.to_datetime(df[\"timestamp\"], unit=\"s\")\n df = df[df[\"datetime\"] < \"2021-06-13 00:00:00\"]\n return df\n\ndf_train = read_csv_strict()\ndisplay(df_train)\n\n\n# Two new features from the competition tutorial\ndef upper_shadow(df):\n return df[\"High\"] - np.maximum(df[\"Close\"], df[\"Open\"])\n\ndef lower_shadow(df):\n return np.minimum(df[\"Close\"], df[\"Open\"]) - df[\"Low\"]\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df,row=False):\n features = []\n keys = [\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]\n\n # df_feat = df[[\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]].copy()\n df_feat = df.copy()\n df_feat[\"Upper_Shadow\"] = upper_shadow(df_feat)\n df_feat[\"Lower_Shadow\"] = lower_shadow(df_feat)\n features += [\"Upper_Shadow\", \"Lower_Shadow\",]\n\n ## Ad dsome more feats\n df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n features += [\"Close/Open\", \"Close-Open\", \"High-Low\", \"High/Low\",]\n\n if row:\n df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n else:\n df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n df_feat[\"High/Mean\"] = df_feat[\"High\"] / df_feat[\"Mean\"]\n df_feat[\"Low/Mean\"] = df_feat[\"Low\"] / df_feat[\"Mean\"]\n df_feat[\"Volume/Count\"] = df_feat[\"Volume\"] / (df_feat[\"Count\"] + 1)\n features += [\"Mean\", \"High/Mean\", \"Low/Mean\", \"Volume/Count\",]\n\n ## possible seasonality, datetime features (unlikely to me meaningful, given very short time-frames)\n ### to do: add cyclical features for seasonality\n times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n if row:\n df_feat[\"hour\"] = times.hour # .dt\n df_feat[\"dayofweek\"] = times.dayofweek \n df_feat[\"day\"] = times.day \n else:\n df_feat[\"hour\"] = times.dt.hour # .dt\n df_feat[\"dayofweek\"] = times.dt.dayofweek \n df_feat[\"day\"] = times.dt.day \n #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True) # keep original epoch time, drop string\n \n df_feat = df_feat[keys + features]\n \n return df_feat\n\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n df = df_train[df_train[\"Asset_ID\"] == asset_id]\n \n # TODO: Try different features here!\n df_proc = get_features(df)\n df_proc[\"y\"] = df[\"Target\"]\n #df_proc = df_proc.dropna(how=\"any\")\n df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n \n X = df_proc.drop(\"y\", axis=1)\n y = df_proc[\"y\"]\n \n # TODO: Try different models here!\n model = LGBMRegressor(\n n_estimators=1500, # 1500,\n num_leaves=700,\n learning_rate=0.09,\n random_state=71,\n )\n model.fit(X, y)\n return X, y, model\n\n\n%%time\nXs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details[\"Asset_ID\"], df_asset_details[\"Asset_Name\"]):\n print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)\n Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model\n\n\n%%time\n# Check the model interface\nx = get_features(df_train.iloc[1], row=True)\ny_pred = models[0].predict([x])\ny_pred[0]\n\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n for j , row in df_test.iterrows():\n \n model = models[row[\"Asset_ID\"]]\n x_test = get_features(row, row=True)\n y_pred = model.predict([x_test])[0]\n \n df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"] = y_pred\n \n \n # Print just one sample row to get a feeling of what it looks like\n if i == 0 and j == 0:\n display(x_test)\n\n # Display the first prediction dataframe\n if i == 0:\n display(df_pred)\n\n # Send submissions\n env.predict(df_pred)\n```\n\nYours sincerely.", "votes": null }, { "id": "1575010", "postDate": "11/08/2021 05:10:08", "content": "My submission takes 1 minute to complete in \"kaggle kernel\", and it also has the 9-hour timeout error. More importantly, I pretrained models so my submission does not spend time on fitting models in each loop.
", "rawMarkdown": "My submission takes 1 minute to complete in \"kaggle kernel\", and it also has the 9-hour timeout error. More importantly, I pretrained models so my submission does not spend time on fitting models in each loop.", "votes": null }, { "id": "1575027", "postDate": "11/08/2021 05:30:52", "content": "Thanks for the info.
\nWhat a surprise that a code that takes only a minute to predict will error out at the 9 hour limit.
\nThat is very troubling. :(
\nI will continue to look for ways to solve these problems.
I suspect that there is something strange going on with the environment during test time.. I get these errors as well from time to time..
", "rawMarkdown": "I suspect that there is something strange going on with the environment during test time.. I get these errors as well from time to time..", "votes": null }, { "id": "1575415", "postDate": "11/08/2021 11:38:12", "content": "Thank you for the info.
\nIn my case, it worked at first, but after that, all I got was an error.
I'm also getting this timeout error in submission when my notebook only takes 1 minute to run in Kaggle kernel. Are there any solution to this so far? Thanks!
", "rawMarkdown": "I'm also getting this timeout error in submission when my notebook only takes 1 minute to run in Kaggle kernel. Are there any solution to this so far? Thanks!", "votes": null }, { "id": "1596711", "postDate": "11/26/2021 18:50:18", "content": "I got timeouts while using 'print' or 'display' in 'iterrows()' loop. Weird, right?
", "rawMarkdown": "I got timeouts while using 'print' or 'display' in 'iterrows()' loop. Weird, right?", "votes": null }, { "id": "1596855", "postDate": "11/26/2021 22:58:12", "content": "Unfortunately, we do not yet have a fundamental solution to this problem. Hmmm.
", "rawMarkdown": "Unfortunately, we do not yet have a fundamental solution to this problem. Hmmm.", "votes": null }, { "id": "1597439", "postDate": "11/27/2021 14:00:32", "content": "I am facing the same issue as you guys , notebooks very simple and fast then submission always got time out …
", "rawMarkdown": "I am facing the same issue as you guys , notebooks very simple and fast then submission always got time out ...", "votes": null }, { "id": "1597601", "postDate": "11/27/2021 17:28:41", "content": "this competition just peek your ideas!!!!
", "rawMarkdown": "this competition just peek your ideas!!!!", "votes": null }, { "id": "1599436", "postDate": "11/29/2021 12:42:25", "content": "Facing same issue, is there any solution to this ?
", "rawMarkdown": "Facing same issue, is there any solution to this ?", "votes": null }, { "id": "1599441", "postDate": "11/29/2021 12:48:48", "content": "Unfortunately, we do not yet have a fundamental solution to this problem. :(
", "rawMarkdown": "Unfortunately, we do not yet have a fundamental solution to this problem. :(", "votes": null }, { "id": "1599445", "postDate": "11/29/2021 12:51:02", "content": "Very bad. 🤕🤕
", "rawMarkdown": "Very bad. 🤕🤕", "votes": null }, { "id": "1604716", "postDate": "12/03/2021 16:28:41", "content": "One guess here, someone calculated that if we are to predict 3 month of test data during submission, we need to call \"predict\" function 90 days * 24 hrs * 60mins = 129600 times (we get one minute of data each per iteration). With the 9 hour limit (32400 seconds), if one prediction takes longer than 0.25 second then it will run out of time.
\nI did some experiments, and it indeed showed that when my prediction iteration spend slightly long than 0.25 second, I get Timeout when submission.
", "rawMarkdown": "One guess here, someone calculated that if we are to predict 3 month of test data during submission, we need to call \"predict\" function 90 days * 24 hrs * 60mins = 129600 times (we get one minute of data each per iteration). With the 9 hour limit (32400 seconds), if one prediction takes longer than 0.25 second then it will run out of time. \n\nI did some experiments, and it indeed showed that when my prediction iteration spend slightly long than 0.25 second, I get Timeout when submission.", "votes": null }, { "id": "1605105", "postDate": "12/04/2021 00:23:50", "content": "OMG…. :(
", "rawMarkdown": "OMG.... :(", "votes": null }, { "id": "1605312", "postDate": "12/04/2021 06:15:55", "content": "Try to remove all 'display() and print()' from 'iterrows' loop. Also try to run in GPU mode. Worked for me.
", "rawMarkdown": "Try to remove all 'display() and print()' from 'iterrows' loop. Also try to run in GPU mode. Worked for me.", "votes": null }, { "id": "1605319", "postDate": "12/04/2021 06:26:24", "content": "Thank you very much. That information is very helpful.
\nI will try to remove all \"display() and print()\" from \"iterrows\" loop.
\"Remove all 'display()' and 'print()' from the 'iterrows' loop.\"
\nThat's the best solution!
\nYeah!! :)
My submission takes 1 minute to complete in \"kaggle kernel\", and it also has the 9-hour timeout error. More importantly, I pretrained models so my submission does not spend time on fitting models in each loop.
", "votes": null, "replies": [ { "id": 1575027, "author_name": "whitecat0022", "author_url": "", "post_date": "11/08/2021 05:30:52", "content": "Thanks for the info.
\nWhat a surprise that a code that takes only a minute to predict will error out at the 9 hour limit.
\nThat is very troubling. :(
\nI will continue to look for ways to solve these problems.
I suspect that there is something strange going on with the environment during test time.. I get these errors as well from time to time..
", "votes": null, "replies": [ { "id": 1575415, "author_name": "whitecat0022", "author_url": "", "post_date": "11/08/2021 11:38:12", "content": "Thank you for the info.
\nIn my case, it worked at first, but after that, all I got was an error.
I'm also getting this timeout error in submission when my notebook only takes 1 minute to run in Kaggle kernel. Are there any solution to this so far? Thanks!
", "votes": null, "replies": [ { "id": 1596855, "author_name": "whitecat0022", "author_url": "", "post_date": "11/26/2021 22:58:12", "content": "Unfortunately, we do not yet have a fundamental solution to this problem. Hmmm.
", "votes": null, "replies": [] }, { "id": 1604716, "author_name": "minshipeng1216", "author_url": "", "post_date": "12/03/2021 16:28:41", "content": "One guess here, someone calculated that if we are to predict 3 month of test data during submission, we need to call \"predict\" function 90 days * 24 hrs * 60mins = 129600 times (we get one minute of data each per iteration). With the 9 hour limit (32400 seconds), if one prediction takes longer than 0.25 second then it will run out of time.
\nI did some experiments, and it indeed showed that when my prediction iteration spend slightly long than 0.25 second, I get Timeout when submission.
", "votes": null, "replies": [] }, { "id": 1605105, "author_name": "whitecat0022", "author_url": "", "post_date": "12/04/2021 00:23:50", "content": "OMG…. :(
", "votes": null, "replies": [] } ] }, { "id": 1596711, "author_name": "kxreev", "author_url": "", "post_date": "11/26/2021 18:50:18", "content": "I got timeouts while using 'print' or 'display' in 'iterrows()' loop. Weird, right?
", "votes": null, "replies": [] }, { "id": 1597439, "author_name": "gianpaolobulleddu", "author_url": "", "post_date": "11/27/2021 14:00:32", "content": "I am facing the same issue as you guys , notebooks very simple and fast then submission always got time out …
", "votes": null, "replies": [] }, { "id": 1597601, "author_name": "dragonzhang", "author_url": "", "post_date": "11/27/2021 17:28:41", "content": "this competition just peek your ideas!!!!
", "votes": null, "replies": [] }, { "id": 1599436, "author_name": "kartik2khandelwal", "author_url": "", "post_date": "11/29/2021 12:42:25", "content": "Facing same issue, is there any solution to this ?
", "votes": null, "replies": [ { "id": 1599441, "author_name": "whitecat0022", "author_url": "", "post_date": "11/29/2021 12:48:48", "content": "Unfortunately, we do not yet have a fundamental solution to this problem. :(
", "votes": null, "replies": [] }, { "id": 1599445, "author_name": "kartik2khandelwal", "author_url": "", "post_date": "11/29/2021 12:51:02", "content": "Very bad. 🤕🤕
", "votes": null, "replies": [] } ] }, { "id": 1605312, "author_name": "kxreev", "author_url": "", "post_date": "12/04/2021 06:15:55", "content": "Try to remove all 'display() and print()' from 'iterrows' loop. Also try to run in GPU mode. Worked for me.
", "votes": null, "replies": [ { "id": 1605319, "author_name": "whitecat0022", "author_url": "", "post_date": "12/04/2021 06:26:24", "content": "Thank you very much. That information is very helpful.
\nI will try to remove all \"display() and print()\" from \"iterrows\" loop.
\"Remove all 'display()' and 'print()' from the 'iterrows' loop.\"
\nThat's the best solution!
\nYeah!! :)