{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport gc\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-05T07:49:12.960121Z","iopub.execute_input":"2021-11-05T07:49:12.960981Z","iopub.status.idle":"2021-11-05T07:49:12.988395Z","shell.execute_reply.started":"2021-11-05T07:49:12.960866Z","shell.execute_reply":"2021-11-05T07:49:12.987490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Train Data","metadata":{}},{"cell_type":"code","source":"TRAINING = True\nDEBUG = False\n\ntrain_data = pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\")\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2021-11-05T07:49:12.990188Z","iopub.execute_input":"2021-11-05T07:49:12.990529Z","iopub.status.idle":"2021-11-05T07:50:06.482618Z","shell.execute_reply.started":"2021-11-05T07:49:12.990491Z","shell.execute_reply":"2021-11-05T07:50:06.481842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details = pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\")\nasset_details.sort_values(by=['Asset_ID'], inplace=True)\nasset_details","metadata":{"execution":{"iopub.status.busy":"2021-11-05T07:50:06.484180Z","iopub.execute_input":"2021-11-05T07:50:06.484479Z","iopub.status.idle":"2021-11-05T07:50:06.506389Z","shell.execute_reply.started":"2021-11-05T07:50:06.484440Z","shell.execute_reply":"2021-11-05T07:50:06.505716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Data Cleaning Function","metadata":{}},{"cell_type":"code","source":"def fix_train_test_data(df, debug=False):\n    df['order1'] = range(df.shape[0])\n    df['order2'] = df.timestamp\n    \n    fixed_data = pd.DataFrame(columns=[col for col in df.columns if col != 'timestamp'])\n    for asset_id in range(14):\n        if debug:\n            print(\"-\"*81)\n            print(\"For Asset_ID:\", asset_id)\n            print()\n        \n        temp = df[df.Asset_ID == asset_id].set_index('timestamp')\n        \n        if debug:\n            print(\"Total number of NaN:\", sum(temp.isna().sum()))\n            \n        temp.fillna(method='ffill', inplace=True)\n        temp.fillna(method='bfill', inplace=True) #For edge cases\n        \n        if debug:\n            print(\"NaN after fill:\", sum(temp.isna().sum()))\n            print()\n            print(\"Before fixing gaps:\")\n            display((temp.index[1:]-temp.index[:-1]).value_counts().head())\n     \n        temp = temp.reindex(range(temp.index[0],temp.index[-1]+60,60),method='pad')\n        \n        if debug:\n            print()\n            print(\"After fixing gaps:\")\n            display((temp.index[1:]-temp.index[:-1]).value_counts().head())\n    \n        fixed_data = pd.concat([fixed_data, temp])\n        if debug:\n            print(\"-\"*81)\n            print(\"Total size:\", fixed_data.shape)\n            \n    return fixed_data.sort_values(by=['order1', 'order2'])\n\nfixed_data = fix_train_test_data(train_data, debug=DEBUG)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-11-05T07:50:06.508540Z","iopub.execute_input":"2021-11-05T07:50:06.508978Z","iopub.status.idle":"2021-11-05T07:51:52.063118Z","shell.execute_reply.started":"2021-11-05T07:50:06.508940Z","shell.execute_reply":"2021-11-05T07:51:52.062123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fixed_data","metadata":{"execution":{"iopub.status.busy":"2021-11-05T07:51:52.064809Z","iopub.execute_input":"2021-11-05T07:51:52.065089Z","iopub.status.idle":"2021-11-05T07:51:52.090339Z","shell.execute_reply.started":"2021-11-05T07:51:52.065050Z","shell.execute_reply":"2021-11-05T07:51:52.089562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check Target Data Fluctuation","metadata":{}},{"cell_type":"code","source":"temp = fixed_data[fixed_data.Asset_ID == 0]\nplt.plot(temp.index, temp.Target, c='#00aa00', alpha=0.75)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T07:51:52.091877Z","iopub.execute_input":"2021-11-05T07:51:52.092154Z","iopub.status.idle":"2021-11-05T07:51:58.527386Z","shell.execute_reply.started":"2021-11-05T07:51:52.092118Z","shell.execute_reply":"2021-11-05T07:51:58.526692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Data","metadata":{}},{"cell_type":"code","source":"test_data = pd.read_csv(\"../input/g-research-crypto-forecasting/example_test.csv\")\ntest_data.head(7)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T07:51:58.528535Z","iopub.execute_input":"2021-11-05T07:51:58.529403Z","iopub.status.idle":"2021-11-05T07:51:58.552191Z","shell.execute_reply.started":"2021-11-05T07:51:58.529360Z","shell.execute_reply":"2021-11-05T07:51:58.551522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fixed_test = fix_train_test_data(test_data, debug=DEBUG)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-11-05T07:51:58.553424Z","iopub.execute_input":"2021-11-05T07:51:58.553698Z","iopub.status.idle":"2021-11-05T07:51:58.592274Z","shell.execute_reply.started":"2021-11-05T07:51:58.553663Z","shell.execute_reply":"2021-11-05T07:51:58.591654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with pd.option_context('display.max_rows',10):\n    display(fixed_test)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T07:51:58.593379Z","iopub.execute_input":"2021-11-05T07:51:58.593902Z","iopub.status.idle":"2021-11-05T07:51:58.615049Z","shell.execute_reply.started":"2021-11-05T07:51:58.593862Z","shell.execute_reply":"2021-11-05T07:51:58.614378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Analysis","metadata":{}},{"cell_type":"code","source":"if not TRAINING:\n    fig, ax = plt.subplots(14, 3, sharex=True, figsize=(24, 32))\n\n    for i, lab in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n        temp = fixed_data[fixed_data.Asset_ID == i]\n        ax[i, 0].title.set_text(f'{lab} Open Price:')\n        ax[i, 0].plot(temp.index, temp.Open, c=\"#0000aa\", alpha=0.75)\n        ax[i, 1].title.set_text(f'{lab} Close Price:')\n        ax[i, 1].plot(temp.index, temp.Close, c=\"#aa0000\", alpha=0.75)\n        ax[i, 2].title.set_text(f'{lab} Price Gap:')\n        ax[i, 2].plot(temp.index, temp.Close - temp.Open, c=\"#00aa00\", alpha=0.75)\n        del temp\n        gc.collect()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T07:51:58.617421Z","iopub.execute_input":"2021-11-05T07:51:58.618118Z","iopub.status.idle":"2021-11-05T07:51:58.626594Z","shell.execute_reply.started":"2021-11-05T07:51:58.618078Z","shell.execute_reply":"2021-11-05T07:51:58.625784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not TRAINING:\n    fig, ax = plt.subplots(14, 3, sharex=True, figsize=(24, 32))\n\n    for i, lab in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n        temp = fixed_data[fixed_data.Asset_ID == i]\n        ax[i, 0].title.set_text(f'{lab} Minute Lowest Price:')\n        ax[i, 0].plot(temp.index, temp.Low, c=\"#0000aa\", alpha=0.75)\n        ax[i, 1].title.set_text(f'{lab} Minute Highest Price:')\n        ax[i, 1].plot(temp.index, temp.High, c=\"#aa0000\", alpha=0.75)\n        ax[i, 2].title.set_text(f'{lab} Minute Gap:')\n        ax[i, 2].plot(temp.index, temp.High - temp.Low, c=\"#00aa00\", alpha=0.75)\n        del temp\n        gc.collect()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T07:51:58.627710Z","iopub.execute_input":"2021-11-05T07:51:58.628388Z","iopub.status.idle":"2021-11-05T07:51:58.639406Z","shell.execute_reply.started":"2021-11-05T07:51:58.628351Z","shell.execute_reply":"2021-11-05T07:51:58.638674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Creation","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, LSTM, Dropout, Reshape, BatchNormalization\n\nnode_size = 128\ndef create_model(n_features):\n    model = Sequential()\n    \n    #model.add(Dense(units=node_size*2, activation='sigmoid', input_shape=(n_features,)))\n    model.add(Reshape((n_features, 1), input_shape=(n_features,)))\n    \n    model.add(LSTM(units = node_size, kernel_initializer='random_normal', return_sequences = True))\n    model.add(Dropout(0.1))\n\n    model.add(LSTM(units = node_size, kernel_initializer='random_normal', return_sequences = True))\n    model.add(Dropout(0.1))\n    \n    model.add(BatchNormalization())\n\n    model.add(LSTM(units = node_size, kernel_initializer='random_normal', return_sequences = True))\n    model.add(Dropout(0.1))\n\n    model.add(LSTM(units = node_size, kernel_initializer='random_normal'))\n    model.add(Dropout(0.1))\n    \n    model.add(BatchNormalization())\n    \n    model.add(Dense(units = 1))\n    \n    optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n    model.compile(optimizer = optimizer, loss = 'mean_squared_error')\n    return model\nmodel = create_model(7)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T07:55:25.708377Z","iopub.execute_input":"2021-11-05T07:55:25.708899Z","iopub.status.idle":"2021-11-05T07:55:26.961234Z","shell.execute_reply.started":"2021-11-05T07:55:25.708846Z","shell.execute_reply":"2021-11-05T07:55:26.960517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\ntrain_columns = [\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]\n\nif TRAINING:\n    for i, lab in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n        print(f\"Training for {lab}:\")\n        x_train = fixed_data[fixed_data.Asset_ID == i]\n        y_train = x_train.Target\n        x_train = x_train[train_columns]\n\n        batch_size = 256\n        epochs = 10\n        model = create_model(len(train_columns))\n        model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, verbose=1)\n        models[lab] = model\n        model.save(f\"{lab}_trained_weights.hdf5\")\n        print(f\"Finished training for {lab}.\")","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-11-05T07:55:33.560389Z","iopub.execute_input":"2021-11-05T07:55:33.560994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\n\nif TRAINING:\n    env = gresearch_crypto.make_env()\n    iter_test = env.iter_test()\n\n    for i, (df_test, df_pred) in enumerate(iter_test):\n        for j , row in df_test.iterrows():\n\n            model = models[asset_details[asset_details.Asset_ID == 0].Asset_Name.values[0]]\n            x_test = row[train_columns]\n            y_pred = model.predict(x_test.values.reshape(1, -1))[0]\n\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n\n            # Print just one sample row to get a feeling of what it looks like\n            if i == 0 and j == 0:\n                display(x_test)\n\n        # Display the first prediction dataframe\n        if i == 0:\n            display(df_pred)\n\n        # Send submissions\n        env.predict(df_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, lab in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    temp = fixed_data[fixed_data.Asset_ID == i][:100]\n    preds = models[lab].predict(temp[train_columns])\n    plt.title(f\"{lab} Target vs Prediction:\")\n    plt.plot(temp.index, temp.Target, c='#00aa00')\n    plt.plot(temp.index, preds, c='#bb0000')\n    plt.legend(['Target', \"Prediction\"])\n    plt.show()\n    plt.savefig(f'{lab}_compare_target.png')\n    plt.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To be continued...","metadata":{}}]}