{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import log_loss\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.model_selection import KFold\n\nimport gresearch_crypto\n\n# Warningの無効化\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\n# データフレームcolumの全表示\npd.set_option(\"display.max_columns\", None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        # else:\n            # df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(r\"../input/g-research-crypto-forecasting/asset_details.csv\").sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_strict(file_name=\"/kaggle/input/g-research-crypto-forecasting/train.csv\"):\n    df = pd.read_csv(file_name).pipe(reduce_mem_usage)\n    df[\"datetime\"] = pd.to_datetime(df[\"timestamp\"], unit=\"s\")\n    df = df[df[\"datetime\"] < \"2021-06-13 00:00:00\"]\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = read_csv_strict()\ndf_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"markdown","source":"## Utility functions to train a model for one asset","metadata":{}},{"cell_type":"code","source":"# technical indicators\ndef RSI(close: pd.DataFrame, period: int = 14) -> pd.Series:\n    # https://gist.github.com/jmoz/1f93b264650376131ed65875782df386\n    \"\"\"See source https://github.com/peerchemist/finta\n    and fix https://www.tradingview.com/wiki/Talk:Relative_Strength_Index_(RSI)\n    Relative Strength Index (RSI) is a momentum oscillator that measures the speed and change of price movements.\n    RSI oscillates between zero and 100. Traditionally, and according to Wilder, RSI is considered overbought when above 70 and oversold when below 30.\n    Signals can also be generated by looking for divergences, failure swings and centerline crossovers.\n    RSI can also be used to identify the general trend.\"\"\"\n\n    delta = close.diff()\n\n    up, down = delta.copy(), delta.copy()\n    up[up < 0] = 0\n    down[down > 0] = 0\n\n    _gain = up.ewm(com=(period - 1), min_periods=period).mean()\n    _loss = down.abs().ewm(com=(period - 1), min_periods=period).mean()\n\n    RS = _gain / _loss\n    return pd.Series(100 - (100 / (1 + RS)))\n\ndef EMA1(x, n):\n    \"\"\"\n    https://qiita.com/MuAuan/items/b08616a841be25d29817\n    \"\"\"\n    a= 2/(n+1)\n    return pd.Series(x).ewm(alpha=a).mean()\n\ndef MACD(close : pd.DataFrame, span1=12, span2=26, span3=9):\n    \"\"\"\n    Compute MACD\n    # https://www.learnpythonwithrune.org/pandas-calculate-the-moving-average-convergence-divergence-macd-for-a-stock/\n    \"\"\"\n    exp1 = EMA1(close, span1)\n    exp2 = EMA1(close, span2)\n    macd = 100 * (exp1 - exp2) / exp2\n    signal = EMA1(macd, span3)\n\n    return macd, signal","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df[\"High\"] - np.maximum(df[\"Close\"], df[\"Open\"])\n\ndef lower_shadow(df):\n    return np.minimum(df[\"Close\"], df[\"Open\"]) - df[\"Low\"]\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df,row=False):\n    features = []\n    keys = [\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]\n\n    # df_feat = df[[\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\", \"VWAP\"]].copy()\n    df_feat = df.copy()\n    df_feat[\"Upper_Shadow\"] = upper_shadow(df_feat)\n    df_feat[\"Lower_Shadow\"] = lower_shadow(df_feat)\n    features += [\"Upper_Shadow\", \"Lower_Shadow\",]\n\n    ## Ad dsome more feats\n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    features += [\"Close/Open\", \"Close-Open\", \"High-Low\", \"High/Low\",]\n\n    if row:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    df_feat[\"High/Mean\"] = df_feat[\"High\"] / df_feat[\"Mean\"]\n    df_feat[\"Low/Mean\"] = df_feat[\"Low\"] / df_feat[\"Mean\"]\n    df_feat[\"Volume/Count\"] = df_feat[\"Volume\"] / (df_feat[\"Count\"] + 1)\n    features += [\"Mean\", \"High/Mean\", \"Low/Mean\", \"Volume/Count\",]\n\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dayofweek \n        df_feat[\"day\"] = times.day \n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dt.dayofweek \n        df_feat[\"day\"] = times.dt.day \n    #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True)  # keep original epoch time, drop string\n    \n    if row:\n        df_feat[\"Median\"] = df_feat[[\"Open\", \"High\", \"Low\", \"Close\"]].median()\n    else:\n        df_feat[\"Median\"] = df_feat[[\"Open\", \"High\", \"Low\", \"Close\"]].median(axis=1)\n    df_feat[\"High/Median\"] = df_feat[\"High\"] / df_feat[\"Median\"]\n    df_feat[\"Low/Median\"] = df_feat[\"Low\"] / df_feat[\"Median\"]\n    features += [\"Median\", \"High/Median\", \"Low/Median\",]\n\n\n    df_feat[\"Log_n_Close\"] = np.log(df_feat[\"Close\"])\n    features += [\"Log_n_Close\",]\n\n    for col in ['Open', 'High', 'Low', 'Close', 'VWAP']:\n        df_feat[f\"Log_1p_{col}\"] = np.log1p(df_feat[col])\n        features += [f\"Log_1p_{col}\",]\n\n    # 基準線\n    #max26 = df_feat[\"High\"].rolling(window=26).max()\n    #min26 = df_feat[\"Low\"].rolling(window=26).min()\n    #df_feat[\"basic_line\"] = (max26 + min26) / 2\n    #features += [\"basic_line\",]\n    \n    # 転換線\n    #high9 = df_feat[\"High\"].rolling(window=9).max()\n    #low9 = df_feat[\"Low\"].rolling(window=9).min()\n    #df_feat[\"turn_line\"] = (high9 + low9) / 2\n    #features += [\"turn_line\",]\n\n    # RSI\n    #df_feat[\"RSI\"] = RSI(df_feat[\"Close\"], 14)\n\n    # MACD\n    macd, macd_signal = MACD(df_feat[\"Close\"], 12, 26, 9) \n    df_feat[\"MACD\"] = macd\n    df_feat[\"MACD_signal\"] = macd_signal\n    features += [\"MACD\", \"MACD_signal\",]\n    \n    df_feat = df_feat[keys + features]\n    \n    return df_feat","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n   \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc[\"y\"] = df[\"Target\"]\n    #df_proc = df_proc.dropna(how=\"any\")\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n\n    # -----------------------------------\n    # Stratified K-Fold\n    # -----------------------------------\n    # StratifiedKFoldクラスを用いて層化抽出による分割を行う\n    kf = KFold(n_splits=4, shuffle=False, random_state=71)    # 時系列順に並んだデータのためshuffle=Falseとする\n    for tr_idx, va_idx in kf.split(X, y):\n        tr_x, va_x = X.iloc[tr_idx], X.iloc[va_idx]\n        tr_y, va_y = y.iloc[tr_idx], y.iloc[va_idx]\n        # Modelクラスは、fitで学習し、predictで予測値の確率を出力する\n\n        # データのスケーリング\n        scaler = RobustScaler()\n        tr_x = scaler.fit_transform(tr_x)\n        va_x = scaler.transform(va_x)\n        #test_x = scaler.transform(test_x)\n\n        # 線形モデルの構築・学習\n        model = Ridge(alpha=1.0)\n        model.fit(tr_x, tr_y)\n\n    return tr_x, tr_y, model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loop over all assets","metadata":{}},{"cell_type":"code","source":"%%time\nXs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details[\"Asset_ID\"], df_asset_details[\"Asset_Name\"]):\n    print(f\"Training model for  {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)\n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Check the model interface\nx = get_features(df_train.iloc[1], row=True)\ny_pred = models[0].predict([x])\ny_pred[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict & submit\n\nReferences: [Detailed API Introduction](https://www.kaggle.com/sohier/detailed-api-introduction)\n\nSomething that helped me understand this iterator was adding a pdb checkpoint inside of the for loop:\n\n```python\nimport pdb; pdb.set_trace()\n```\n\nSee [Python Debugging With Pdb](https://realpython.com/python-debugging-pdb/) if you want to use it and you don't know how to.\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T20:57:49.349459Z","iopub.status.idle":"2021-11-02T20:57:49.349757Z","shell.execute_reply.started":"2021-11-02T20:57:49.349596Z","shell.execute_reply":"2021-11-02T20:57:49.349613Z"}}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        model = models[row[\"Asset_ID\"]]\n        x_test = get_features(row, row=True)\n        y_pred = model.predict([x_test])[0]\n        \n        df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"] = y_pred\n        \n        \n        # Print just one sample row to get a feeling of what it looks like\n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}