{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-03T12:41:17.122733Z","iopub.execute_input":"2021-12-03T12:41:17.123025Z","iopub.status.idle":"2021-12-03T12:41:17.139746Z","shell.execute_reply.started":"2021-12-03T12:41:17.122992Z","shell.execute_reply":"2021-12-03T12:41:17.138845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the Train set\n\ndf_train = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv',nrows = 10000).astype(np.uint8,errors='ignore')\ndf_asset = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv')\ndf_sub = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/example_sample_submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.142157Z","iopub.execute_input":"2021-12-03T12:41:17.142758Z","iopub.status.idle":"2021-12-03T12:41:17.205981Z","shell.execute_reply.started":"2021-12-03T12:41:17.1427Z","shell.execute_reply":"2021-12-03T12:41:17.205323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.207647Z","iopub.execute_input":"2021-12-03T12:41:17.20825Z","iopub.status.idle":"2021-12-03T12:41:17.229505Z","shell.execute_reply.started":"2021-12-03T12:41:17.208195Z","shell.execute_reply":"2021-12-03T12:41:17.228644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.231092Z","iopub.execute_input":"2021-12-03T12:41:17.231678Z","iopub.status.idle":"2021-12-03T12:41:17.238306Z","shell.execute_reply.started":"2021-12-03T12:41:17.231634Z","shell.execute_reply":"2021-12-03T12:41:17.23736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.240876Z","iopub.execute_input":"2021-12-03T12:41:17.241334Z","iopub.status.idle":"2021-12-03T12:41:17.259404Z","shell.execute_reply.started":"2021-12-03T12:41:17.241294Z","shell.execute_reply":"2021-12-03T12:41:17.258746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.26065Z","iopub.execute_input":"2021-12-03T12:41:17.261603Z","iopub.status.idle":"2021-12-03T12:41:17.317314Z","shell.execute_reply.started":"2021-12-03T12:41:17.261558Z","shell.execute_reply":"2021-12-03T12:41:17.316452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['Asset_ID'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.318396Z","iopub.execute_input":"2021-12-03T12:41:17.319094Z","iopub.status.idle":"2021-12-03T12:41:17.326858Z","shell.execute_reply.started":"2021-12-03T12:41:17.31906Z","shell.execute_reply":"2021-12-03T12:41:17.32621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assetnames = df_train['Asset_ID'].unique()\nassetnames = np.sort(assetnames,kind='quicksort')\nprint(assetnames)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.328052Z","iopub.execute_input":"2021-12-03T12:41:17.328942Z","iopub.status.idle":"2021-12-03T12:41:17.33807Z","shell.execute_reply.started":"2021-12-03T12:41:17.328905Z","shell.execute_reply":"2021-12-03T12:41:17.337465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Find missing data\ntotal = df_train.isnull().sum().sort_values(ascending = False)\npercent = (df_train.isnull().sum()/ df_train.isnull().count()).sort_values(ascending = False)*100\nmissing_data = pd.concat([total,percent],axis=1, keys = ['Total','Percent'])\nmissing_data","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.339275Z","iopub.execute_input":"2021-12-03T12:41:17.339778Z","iopub.status.idle":"2021-12-03T12:41:17.363068Z","shell.execute_reply.started":"2021-12-03T12:41:17.339744Z","shell.execute_reply":"2021-12-03T12:41:17.362462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Drop Few missing values in target feature\ndf_train = df_train.drop(df_train.loc[df_train['Target'].isnull()].index)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.364438Z","iopub.execute_input":"2021-12-03T12:41:17.364853Z","iopub.status.idle":"2021-12-03T12:41:17.371369Z","shell.execute_reply.started":"2021-12-03T12:41:17.364823Z","shell.execute_reply":"2021-12-03T12:41:17.370653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['Date'] = pd.to_datetime(df_train['timestamp'],format =\"%Y/%m/%d\")","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.372746Z","iopub.execute_input":"2021-12-03T12:41:17.37297Z","iopub.status.idle":"2021-12-03T12:41:17.385861Z","shell.execute_reply.started":"2021-12-03T12:41:17.372942Z","shell.execute_reply":"2021-12-03T12:41:17.385218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv')\n","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:41:17.387184Z","iopub.execute_input":"2021-12-03T12:41:17.387968Z","iopub.status.idle":"2021-12-03T12:42:17.246355Z","shell.execute_reply.started":"2021-12-03T12:41:17.387921Z","shell.execute_reply":"2021-12-03T12:42:17.245568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nbtc['timestamp'] = [datetime.fromtimestamp(x) for x in btc['timestamp']]\nbtc = btc.set_index('timestamp')\nbtc_daily = btc.resample(\"24H\").mean()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:42:44.316865Z","iopub.execute_input":"2021-12-03T12:42:44.317423Z","iopub.status.idle":"2021-12-03T12:43:13.332073Z","shell.execute_reply.started":"2021-12-03T12:42:44.317386Z","shell.execute_reply":"2021-12-03T12:43:13.33136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### function to impute missing values using interpolation ###\n\ndef fill_missing(df):\n    l = ['Count', 'Open', 'High', 'Low',\n       'Close', 'Volume', 'VWAP','Target']\n    for p in l:\n        df[p] = df[p].interpolate()\n\n    print(df.head())\n    print(df.isnull().sum())\n    \nfill_missing(btc_daily)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:32:35.919559Z","iopub.execute_input":"2021-12-03T13:32:35.919893Z","iopub.status.idle":"2021-12-03T13:32:35.941101Z","shell.execute_reply.started":"2021-12-03T13:32:35.919858Z","shell.execute_reply":"2021-12-03T13:32:35.939762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from statsmodels.tsa.seasonal import seasonal_decompose\nimport matplotlib.pyplot as plt\nplt.figure(figsize=(15,12))\nseries = btc_daily.Target\nresult = seasonal_decompose(series, model='additive',period=1)\nresult.plot()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:32:45.328803Z","iopub.execute_input":"2021-12-03T13:32:45.329321Z","iopub.status.idle":"2021-12-03T13:32:48.903322Z","shell.execute_reply.started":"2021-12-03T13:32:45.329284Z","shell.execute_reply":"2021-12-03T13:32:48.902334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Renaming the column names accroding to Prophet's requirements\n\ndaily_data_fb = btc_daily.reset_index()[['timestamp','Target']].rename({'timestamp':'ds','Target':'y'}, axis=1)\ndaily_data_fb.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:33:29.812217Z","iopub.execute_input":"2021-12-03T13:33:29.812833Z","iopub.status.idle":"2021-12-03T13:33:29.826012Z","shell.execute_reply.started":"2021-12-03T13:33:29.812793Z","shell.execute_reply":"2021-12-03T13:33:29.825393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_length = 365\nsplit_date = \"2020-11-01\"\ntrain_filt = daily_data_fb['ds'] <= split_date\ntest_filt = daily_data_fb['ds'] > split_date\n\ntrain_fb = daily_data_fb[train_filt]\ntest_fb = daily_data_fb[test_filt]\n\nprint(\"train data shape :\", train_fb.shape[0] / daily_data_fb.shape[0] * 100)\nprint(\"test data shape :\", test_fb.shape[0] / daily_data_fb.shape[0] * 100)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:33:29.82775Z","iopub.execute_input":"2021-12-03T13:33:29.828208Z","iopub.status.idle":"2021-12-03T13:33:29.841047Z","shell.execute_reply.started":"2021-12-03T13:33:29.828169Z","shell.execute_reply":"2021-12-03T13:33:29.840397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install auto-ts\n","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:33:29.842147Z","iopub.execute_input":"2021-12-03T13:33:29.842685Z","iopub.status.idle":"2021-12-03T13:33:39.549086Z","shell.execute_reply.started":"2021-12-03T13:33:29.842653Z","shell.execute_reply":"2021-12-03T13:33:39.548259Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install dask-xgboost\nfrom auto_ts import auto_timeseries \n","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:33:39.5509Z","iopub.execute_input":"2021-12-03T13:33:39.551223Z","iopub.status.idle":"2021-12-03T13:33:48.256383Z","shell.execute_reply.started":"2021-12-03T13:33:39.551176Z","shell.execute_reply":"2021-12-03T13:33:48.255391Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fb = train_fb\ntrain_fb.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:33:48.259584Z","iopub.execute_input":"2021-12-03T13:33:48.259885Z","iopub.status.idle":"2021-12-03T13:33:48.274443Z","shell.execute_reply.started":"2021-12-03T13:33:48.259846Z","shell.execute_reply":"2021-12-03T13:33:48.273371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fb['ds'] = train_fb['ds'].astype('str')\ntest_fb['ds'] = test_fb['ds'].astype('str')\n","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:33:48.276072Z","iopub.execute_input":"2021-12-03T13:33:48.276366Z","iopub.status.idle":"2021-12-03T13:33:48.299492Z","shell.execute_reply.started":"2021-12-03T13:33:48.276333Z","shell.execute_reply":"2021-12-03T13:33:48.298519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_fb = test_fb\ntest_fb.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:33:48.300849Z","iopub.execute_input":"2021-12-03T13:33:48.301116Z","iopub.status.idle":"2021-12-03T13:33:48.312535Z","shell.execute_reply.started":"2021-12-03T13:33:48.301085Z","shell.execute_reply":"2021-12-03T13:33:48.311515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fb = train_fb.replace(to_replace='None', value=np.nan).dropna()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:33:48.314041Z","iopub.execute_input":"2021-12-03T13:33:48.314694Z","iopub.status.idle":"2021-12-03T13:33:48.326519Z","shell.execute_reply.started":"2021-12-03T13:33:48.314646Z","shell.execute_reply":"2021-12-03T13:33:48.325509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = auto_timeseries(forecast_period=219, score_type='rmse', time_interval='D', model_type='best')\n# model.fit(traindata= train_fb, ts_column=\"ds\", target=\"y\")\n\nautoml_model = auto_timeseries(\n    score_type='rmse',\n    forecast_period=test_length, \n    model_type='best', verbose=0\n)\n\nts_column = 'ds'\ntarget = 'y'\nsep = ','\n\nautoml_model.fit(\n    traindata=train_fb,\n    ts_column=ts_column,\n    target=target,\n    cv=5,\n    sep=',')\n","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:33:48.327859Z","iopub.execute_input":"2021-12-03T13:33:48.328711Z","iopub.status.idle":"2021-12-03T13:34:13.553789Z","shell.execute_reply.started":"2021-12-03T13:33:48.328675Z","shell.execute_reply":"2021-12-03T13:34:13.552706Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate Results","metadata":{}},{"cell_type":"code","source":"y_pred = automl_model.predict(test_fb,model='best')\npredf = pd.DataFrame([test_fb[target].values, y_pred['yhat'].values]).T\npredf.columns = ['Actual','Predicted']\npredf.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:34:13.555313Z","iopub.execute_input":"2021-12-03T13:34:13.555608Z","iopub.status.idle":"2021-12-03T13:34:29.342673Z","shell.execute_reply.started":"2021-12-03T13:34:13.555567Z","shell.execute_reply":"2021-12-03T13:34:29.34166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from auto_ts import print_ts_model_stats\nprint_ts_model_stats(test_fb[target].values, y_pred['yhat'].values,'Best')","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:34:29.344235Z","iopub.execute_input":"2021-12-03T13:34:29.344466Z","iopub.status.idle":"2021-12-03T13:34:29.716277Z","shell.execute_reply.started":"2021-12-03T13:34:29.344439Z","shell.execute_reply":"2021-12-03T13:34:29.715349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Forecast Results","metadata":{}},{"cell_type":"code","source":"# Using Best Model\nfuture_predictions = automl_model.predict(\n    testdata=test_fb\n)  \nfuture_predictions","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:34:29.717996Z","iopub.execute_input":"2021-12-03T13:34:29.718268Z","iopub.status.idle":"2021-12-03T13:34:45.84575Z","shell.execute_reply.started":"2021-12-03T13:34:29.718235Z","shell.execute_reply":"2021-12-03T13:34:45.84438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using specific model\nfuture_predictions = automl_model.predict(\n    testdata=test_fb,\n    model='ML',\n    simple=False\n\n)\nfuture_predictions","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:34:45.846926Z","iopub.execute_input":"2021-12-03T13:34:45.847143Z","iopub.status.idle":"2021-12-03T13:35:01.628902Z","shell.execute_reply.started":"2021-12-03T13:34:45.847104Z","shell.execute_reply":"2021-12-03T13:35:01.628028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\nsubm = test_fb[[target]]\nprint(subm.shape)\nsubm.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:35:01.631274Z","iopub.execute_input":"2021-12-03T13:35:01.631492Z","iopub.status.idle":"2021-12-03T13:35:01.643667Z","shell.execute_reply.started":"2021-12-03T13:35:01.631464Z","shell.execute_reply":"2021-12-03T13:35:01.642743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm['predictions'] = future_predictions['yhat'].values\nsubm[[target,'predictions']].plot(figsize=(15,8))","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:35:01.644851Z","iopub.execute_input":"2021-12-03T13:35:01.645092Z","iopub.status.idle":"2021-12-03T13:35:01.994322Z","shell.execute_reply.started":"2021-12-03T13:35:01.645062Z","shell.execute_reply":"2021-12-03T13:35:01.993246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub['Target'] = test_fb[[target]]\ndf_sub.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:41:54.468785Z","iopub.execute_input":"2021-12-03T13:41:54.469586Z","iopub.status.idle":"2021-12-03T13:41:54.478948Z","shell.execute_reply.started":"2021-12-03T13:41:54.469527Z","shell.execute_reply":"2021-12-03T13:41:54.478121Z"},"trusted":true},"execution_count":null,"outputs":[]}]}