{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nfrom collections import deque\nimport io\nimport os\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.svm import SVR\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler,MinMaxScaler\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import max_error\nfrom sklearn.decomposition import PCA\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import mean_squared_error,make_scorer\nfrom xgboost import XGBRegressor as xgb\nfrom sklearn.cluster import MeanShift, KMeans\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-13T12:47:32.886271Z","iopub.execute_input":"2022-01-13T12:47:32.887053Z","iopub.status.idle":"2022-01-13T12:47:35.601258Z","shell.execute_reply.started":"2022-01-13T12:47:32.886884Z","shell.execute_reply":"2022-01-13T12:47:35.600074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset=pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\ntest1=pd.read_csv('../input/g-research-crypto-forecasting/example_test.csv')#.set_index('timestamp')","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:47:35.603231Z","iopub.execute_input":"2022-01-13T12:47:35.603481Z","iopub.status.idle":"2022-01-13T12:47:35.628324Z","shell.execute_reply.started":"2022-01-13T12:47:35.603451Z","shell.execute_reply":"2022-01-13T12:47:35.627244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\ndf_train=df_train.tail(10000000)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:47:35.629852Z","iopub.execute_input":"2022-01-13T12:47:35.630555Z","iopub.status.idle":"2022-01-13T12:48:39.465362Z","shell.execute_reply.started":"2022-01-13T12:47:35.630500Z","shell.execute_reply":"2022-01-13T12:48:39.462759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#supplemental_train=pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv')\ndf_train.append(pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv'))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:48:39.470995Z","iopub.execute_input":"2022-01-13T12:48:39.472210Z","iopub.status.idle":"2022-01-13T12:48:48.687257Z","shell.execute_reply.started":"2022-01-13T12:48:39.472046Z","shell.execute_reply":"2022-01-13T12:48:48.685621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef analysis(data1,name):\n    def moving_average(x,w):\n        con=np.convolve(x, np.ones(w), 'valid') / w\n        zeros_len=len(x)-len(con)\n        con=np.append(np.zeros(zeros_len),con)\n        return(con)\n    \n    name=name[0].strip().replace(' ','_')\n    data1[['current_pos','%diff','diffhl','%upfromstart','EMA4','EMA24','MACD','MACDSignalLine','Histogram','sma4h','std','upboil','boilint','lowboil',\n           'current_boil_pos','%diffSTD','Log_Ret','Volatility','std24upfromstart','std4upfromstart','std24diffhl','std4diffhl']]=np.nan\n    npclose=np.array(data1['Close'])\n    nphigh=np.array(data1['High'])\n    nplow=np.array(data1['Low'])\n    npopen=np.array(data1['Open'])\n                   \n    data1.loc[:,'current_pos']=((npclose-nplow)/(nphigh-nplow))*100\n    data1.loc[:,'%diff']=((npclose-npopen)/npopen)*100\n    data1.loc[:,'diffhl']=((nphigh-nplow)/nplow)*100\n    data1.loc[:,'%upfromstart']=((nphigh-npopen)/npopen)*100\n    data1.loc[:,'EMA4']= moving_average(npclose,4)\n    data1.loc[:,'EMA24']=moving_average(npclose,24)\n    data1.loc[:,'MACD'] = np.array(data1['EMA24']) - np.array(data1['EMA4'])\n    data1.loc[:,'MACDSignalLine'] = moving_average(data1['MACD'],4)\n    data1.loc[:,'Histogram'] = np.array(data1['MACD']) - np.array(data1['MACDSignalLine'])\n\n    data1.loc[:,'sma4h'] = moving_average(np.array(data1['Close']),24)\n    data1.loc[:,'std'] = data1['Close'].rolling(window=24).std()\n    data1.loc[:,'upboil'] = np.array(data1['sma4h']) + 2 * np.array(data1['std'])\n    data1.loc[:,'lowboil'] = np.array(data1['sma4h']) - 2 * np.array(data1['std'])\n    data1.loc[:,'boilint']=0\n#     data1.loc[data1['Close']<=data1['lowboil'],'boilint']=1\n#     data1.loc[data1['Close']>=data1['upboil'],'boilint']=-1\n    data1.loc[:,'current_boil_pos']=((np.array(data1['Close'])-np.array(data1['lowboil']))/(np.array(data1['upboil'])-np.array(data1['lowboil'])))*100\n    data1.loc[:,'%diffSTD']=np.array(data1['%diff']).std()\n\n    data1.loc[:,'Log_Ret'] = np.log(data1['Close'] / data1['Close'].shift(1))\n    data1.loc[:,'Volatility'] = data1['Log_Ret'].rolling(window=24).std() * np.sqrt(24)\n\n    data1.loc[:,'std24upfromstart'] = data1['%upfromstart'].rolling(window=24).std()\n    data1.loc[:,'std4upfromstart']= data1['%upfromstart'].rolling(window=4).std()\n\n    data1.loc[:,'std24diffhl'] = data1['diffhl'].rolling(window=24).std()\n    data1.loc[:,'std4diffhl'] = data1['diffhl'].rolling(window=4).std()\n    \n    for col in data1.columns:\n        data1.rename(columns={col:col+name},inplace=True)\n    data1=pd.DataFrame(data1)\n    return(data1)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:48:48.690242Z","iopub.execute_input":"2022-01-13T12:48:48.690819Z","iopub.status.idle":"2022-01-13T12:48:48.724990Z","shell.execute_reply.started":"2022-01-13T12:48:48.690745Z","shell.execute_reply":"2022-01-13T12:48:48.723161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def analysing(df_train_2,df_asset):\n    df3=pd.DataFrame()\n    for i in df_train_2['Asset_ID'].unique():\n        #if i<=3:\n            coin_name=df_asset['Asset_Name'].loc[df_asset['Asset_ID']==i].values\n            coin_name_data=df_train_2[df_train_2['Asset_ID']==i].set_index('timestamp')\n            try: \n                coin_name_data.drop(columns=['row_id'],inplace=True)\n            except:\n                pass\n            \n            coin_name_data=analysis(coin_name_data,coin_name)\n                        \n            if df3.empty:\n                df3=coin_name_data\n            else:\n                df3=df3.merge(coin_name_data,how='inner',left_index=True, right_index=True)\n                #df=pd.concat((df,coin_name_data),axis=1,join=\"outer\")\n            del coin_name_data\n    return(df3)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:48:48.727867Z","iopub.execute_input":"2022-01-13T12:48:48.728330Z","iopub.status.idle":"2022-01-13T12:48:48.747012Z","shell.execute_reply.started":"2022-01-13T12:48:48.728260Z","shell.execute_reply":"2022-01-13T12:48:48.745795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def final_df_def(df):\n    final_df=df.copy()\n    try:\n        for time in df_test.timestamp.unique():\n            final_df.drop(index=time,inplace=True)\n    except:print('ok')\n        \n    final_df=final_df.tail(8000)\n    return(final_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:48:48.749772Z","iopub.execute_input":"2022-01-13T12:48:48.750238Z","iopub.status.idle":"2022-01-13T12:48:48.762904Z","shell.execute_reply.started":"2022-01-13T12:48:48.750167Z","shell.execute_reply":"2022-01-13T12:48:48.761641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_targets(df_asset):\n    targets={}\n    for name in df_asset['Asset_Name'].unique():\n        name=name.replace(' ','_')\n        targets[name]=final_df['Target'+name]\n        del final_df['Target'+name]\n    return(targets)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:48:48.765125Z","iopub.execute_input":"2022-01-13T12:48:48.765614Z","iopub.status.idle":"2022-01-13T12:48:48.786848Z","shell.execute_reply.started":"2022-01-13T12:48:48.765419Z","shell.execute_reply":"2022-01-13T12:48:48.785930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rmse(predict, actual):\n    predict = np.array(predict)\n    actual = np.array(actual)\n\n    distance = predict - actual\n\n    square_distance = distance ** 2\n\n    mean_square_distance = square_distance.mean()\n\n    score = np.sqrt(mean_square_distance)\n\n    return score\n\nrmse_score = make_scorer(rmse, greater_is_better = False)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:48:48.788642Z","iopub.execute_input":"2022-01-13T12:48:48.788974Z","iopub.status.idle":"2022-01-13T12:48:48.802687Z","shell.execute_reply.started":"2022-01-13T12:48:48.788936Z","shell.execute_reply":"2022-01-13T12:48:48.801342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_training(cryp,data,targets,models,pca_dict,columns_dict,scaler_dict,best_params):\n    X=data.copy()\n    X['future'+cryp]=targets[cryp]\n    y=targets[cryp]\n    cordata=abs(X.corr(method='pearson')['future'+cryp]).sort_values(ascending=False)\n    X=X[cordata[:20].index]\n    try:\n        X.drop(['future'+cryp], axis=1,inplace=True)\n    except:\n        pass\n\n\n    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.95, test_size=0.05,shuffle=False )\n\n    columns=X.columns\n    pca = PCA(.98)\n    scaler = StandardScaler()\n    X_train=scaler.fit_transform(X_train)\n    X_train=pca.fit_transform(X_train)\n    X_train=pd.DataFrame(X_train) \n    \n#     grid = GridSearchCV(SVR(),{'C': [1],'gamma': [1e-8],'epsilon':[0.001],'kernel': ['rbf']}, cv=10, return_train_score=False, verbose = 0,n_jobs=3,scoring=rmse_score)\n#     grid.fit(X_train,y_train)\n#     pp=grid.best_params_\n#     print(cryp,pp,grid.best_score_)\n    pp={'C': 1, 'gamma': 1e-8, 'kernel': 'rbf','epsilon':0.001}\n    model=SVR()\n    model.set_params(**pp)\n    \n    model=model.fit(X_train,y_train)\n    models[cryp]=model\n    pca_dict[cryp]=pca\n    columns_dict[cryp]=columns\n    scaler_dict[cryp]=scaler\n    best_params[cryp]=pp\n\n    del X,cordata\n    del y\n    return(models,pca_dict,columns_dict,scaler_dict,best_params)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:48:48.806019Z","iopub.execute_input":"2022-01-13T12:48:48.806686Z","iopub.status.idle":"2022-01-13T12:48:48.827286Z","shell.execute_reply.started":"2022-01-13T12:48:48.806643Z","shell.execute_reply":"2022-01-13T12:48:48.826436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n  \n    start_mem = df.memory_usage().sum() / 1024**2\n\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n\n    df[\"Count\"] = df[\"Count\"].astype(np.int16)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    \n\n     \n    return df\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:48:48.828885Z","iopub.execute_input":"2022-01-13T12:48:48.829821Z","iopub.status.idle":"2022-01-13T12:48:48.853106Z","shell.execute_reply.started":"2022-01-13T12:48:48.829766Z","shell.execute_reply":"2022-01-13T12:48:48.852161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models={}\npca_dict={}\ncolumns_dict={}\nscaler_dict={}\nbest_params={}\ndf_train = reduce_mem_usage(df_train)\ndf=analysing(df_train,df_asset)\ndf.replace([np.inf, -np.inf], np.nan, inplace=True)\ndf.fillna(0,inplace=True)\nfinal_df=final_df_def(df)\ntargets=get_targets(df_asset)\n\nfor name in df_asset['Asset_Name'].unique():\n    name=name.replace(' ','_')\n    models,pca_dict,columns_dict,scaler_dict,best_params=model_training(name,final_df,targets,models,pca_dict,columns_dict,scaler_dict,best_params)\ndel final_df\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:48:48.855255Z","iopub.execute_input":"2022-01-13T12:48:48.855923Z","iopub.status.idle":"2022-01-13T12:50:47.456559Z","shell.execute_reply.started":"2022-01-13T12:48:48.855863Z","shell.execute_reply":"2022-01-13T12:50:47.455367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#import time\nimport gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor (df_test, df_pred) in iter_test:\n    df_test=reduce_mem_usage(df_test)\n    df_train=df_train.append(df_test)\n    #start_time = time.time()\n    df1=analysing(df_train.tail(len(df_test)+245),df_asset)\n    #print(\"--- %s seconds ---\" % (time.time() - start_time))\n    df1.replace([np.inf, -np.inf], 0, inplace=True)\n    df1.fillna(0,inplace=True)\n        \n    pred_time=df_test['timestamp'].iloc[0]\n    predict_df=df1.loc[df1.index==pred_time]\n\n    for _, row in df_test.iterrows():\n        try:\n            coin_id=row['Asset_ID']\n            coin=df_asset['Asset_Name'].loc[df_asset['Asset_ID']==coin_id].values[0].replace(' ','_')\n            model=models[coin].set_params(**best_params[coin])\n            pca=pca_dict[coin]\n            predictdf=predict_df.copy()\n            predictdf=predictdf[columns_dict[coin]]\n            scaler=scaler_dict[coin]\n            predictdf=scaler.transform(predictdf)\n            predictdf=pca.transform(predictdf)\n            predictdf=pd.DataFrame(predictdf)\n            prediction=model.predict(predictdf)\n            prediction=round(prediction[0],16)\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = prediction\n        except:\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n\n    df_pred['Target'] = df_pred['Target'].fillna(0)\n    df_pred['Target'].astype('float64')\n    env.predict(df_pred)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T12:50:47.458484Z","iopub.execute_input":"2022-01-13T12:50:47.458838Z","iopub.status.idle":"2022-01-13T12:50:50.783368Z","shell.execute_reply.started":"2022-01-13T12:50:47.458782Z","shell.execute_reply":"2022-01-13T12:50:50.782580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}