{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-12T18:53:17.695371Z","iopub.execute_input":"2021-12-12T18:53:17.696115Z","iopub.status.idle":"2021-12-12T18:53:17.712754Z","shell.execute_reply.started":"2021-12-12T18:53:17.696078Z","shell.execute_reply":"2021-12-12T18:53:17.711841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install kats","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:53:17.714926Z","iopub.execute_input":"2021-12-12T18:53:17.715198Z","iopub.status.idle":"2021-12-12T18:55:57.878657Z","shell.execute_reply.started":"2021-12-12T18:53:17.715166Z","shell.execute_reply":"2021-12-12T18:55:57.877418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom kats.detectors.bocpd import BOCPDetector, BOCPDModelType, TrendChangeParameters, NormalKnownParameters\nfrom kats.detectors.robust_stat_detection import RobustStatDetector\nfrom kats.consts import TimeSeriesData\nimport plotly.express as px\nfrom datetime import datetime\nimport warnings","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:55:57.881033Z","iopub.execute_input":"2021-12-12T18:55:57.881486Z","iopub.status.idle":"2021-12-12T18:56:02.597740Z","shell.execute_reply.started":"2021-12-12T18:55:57.881437Z","shell.execute_reply":"2021-12-12T18:56:02.596979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\nasset_id = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\nasset = asset_id.set_index('Asset_ID').sort_index()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:56:02.598869Z","iopub.execute_input":"2021-12-12T18:56:02.599488Z","iopub.status.idle":"2021-12-12T18:57:03.086191Z","shell.execute_reply.started":"2021-12-12T18:56:02.599445Z","shell.execute_reply":"2021-12-12T18:57:03.085308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The function below does cp detection with kats library**\n## *Parameters:*\n* *data* - data \n* *asset_id* - coin id \n* *prior* - as I use Bayesian online detection method with an assumed nornmal distribution of changepoints prior stands for share of cp among datapoints\n* *feature* - 'Close' or 'Volume\n* *confidence* - filtering cps only with confidence greater than it\n* *scaling* - if True standard sacaling of values, I sometimes prefer to use it for Volume","metadata":{}},{"cell_type":"code","source":"def changepoint_auto(data, asset_id, prior, feature, confidence = 0.75, scaling=True):\n    asset_name = asset.loc[asset_id]['Asset_Name']\n    asset_data = data[data['Asset_ID'] == asset_id].set_index(\"timestamp\")\n    tdf = pd.DataFrame()\n    tdf['time'] = pd.to_datetime(asset_data.index*1000000000)\n    tdf['value'] = asset_data[feature].values\n    if feature == 'Close':\n        th = tdf.resample('D', on='time').value.mean().to_frame()\n    else:\n        th = tdf.resample('D', on='time').value.sum().to_frame()\n    th['time'] = th.index\n    if scaling:\n        sc = StandardScaler()\n        cols_to_scale = ['value']\n        th['value'] = sc.fit_transform(th[cols_to_scale].values)\n    ts = TimeSeriesData(th)    \n    bocpd_detector = BOCPDetector(ts)\n    changepoints = bocpd_detector.detector(\n        model=BOCPDModelType.NORMAL_KNOWN_MODEL,\n        model_parameters = NormalKnownParameters(\n                #cp_prior=0.01, \n                #search_method= 'random',  \n                #known_prec_multiplier = 1.0,\n                empirical=True\n            ),\n        threshold = confidence,\n        changepoint_prior=prior,\n        choose_priors = True,\n        agg_cp = True\n    )   \n    \n    fig = px.line(th.value, title='Daily {0} graph for {1} with changepoints, value standard scaling: {2}'.format(feature, asset_name, scaling))\n    \n    #filetered_cahngepoints = []\n    i = 1\n    for changepoint in changepoints:\n        if i%2==0:\n            color = \"red\"\n        else:\n            color = \"green\"\n        if i==1:\n            point = datetime.strptime(str(changepoint[0].start_time).split('T')[0], \"%Y-%m-%d\")\n        elif i>2:\n            point_p = point\n            point = datetime.strptime(str(changepoint[0].start_time).split('T')[0], \"%Y-%m-%d\")\n            fig.add_vrect(\n                x0 = point_p,\n                x1 = point,\n              fillcolor=color, opacity=0.05, line_width=0)\n        i+=1\n        \n    #print(asset_name, len(changepoints), filetered_cahngepoints)\n    #bocpd_detector.plot(changepoints)\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:03.088860Z","iopub.execute_input":"2021-12-12T18:57:03.089111Z","iopub.status.idle":"2021-12-12T18:57:03.105572Z","shell.execute_reply.started":"2021-12-12T18:57:03.089082Z","shell.execute_reply":"2021-12-12T18:57:03.104535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nchangepoint_auto(data, 0, 5e-3, 'Volume', 0.7, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:03.106930Z","iopub.execute_input":"2021-12-12T18:57:03.107199Z","iopub.status.idle":"2021-12-12T18:57:08.835874Z","shell.execute_reply.started":"2021-12-12T18:57:03.107169Z","shell.execute_reply":"2021-12-12T18:57:08.835228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 0, 5e-3, 'Close', 0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:08.836877Z","iopub.execute_input":"2021-12-12T18:57:08.837116Z","iopub.status.idle":"2021-12-12T18:57:14.647715Z","shell.execute_reply.started":"2021-12-12T18:57:08.837087Z","shell.execute_reply":"2021-12-12T18:57:14.646372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 1, 5e-3, 'Volume', 0.6, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:34:01.706821Z","iopub.execute_input":"2021-12-12T19:34:01.707134Z","iopub.status.idle":"2021-12-12T19:34:06.680249Z","shell.execute_reply.started":"2021-12-12T19:34:01.707100Z","shell.execute_reply":"2021-12-12T19:34:06.679308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 1, 5e-5, 'Close', 0.8, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:19.622259Z","iopub.execute_input":"2021-12-12T18:57:19.622533Z","iopub.status.idle":"2021-12-12T18:57:25.229461Z","shell.execute_reply.started":"2021-12-12T18:57:19.622502Z","shell.execute_reply":"2021-12-12T18:57:25.228598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 2, 5e-4, 'Volume', 0.65, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:25.230926Z","iopub.execute_input":"2021-12-12T18:57:25.231167Z","iopub.status.idle":"2021-12-12T18:57:30.257061Z","shell.execute_reply.started":"2021-12-12T18:57:25.231141Z","shell.execute_reply":"2021-12-12T18:57:30.256112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 2, 5e-5, 'Close', 0.77, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:30.258434Z","iopub.execute_input":"2021-12-12T18:57:30.258679Z","iopub.status.idle":"2021-12-12T18:57:35.720734Z","shell.execute_reply.started":"2021-12-12T18:57:30.258648Z","shell.execute_reply":"2021-12-12T18:57:35.719830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 3, 5e-3, 'Volume', 0.7, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:35.722164Z","iopub.execute_input":"2021-12-12T18:57:35.722514Z","iopub.status.idle":"2021-12-12T18:57:40.036052Z","shell.execute_reply.started":"2021-12-12T18:57:35.722477Z","shell.execute_reply":"2021-12-12T18:57:40.035338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 3, 5e-5, 'Close', 0.7, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:40.037385Z","iopub.execute_input":"2021-12-12T18:57:40.037635Z","iopub.status.idle":"2021-12-12T18:57:45.850583Z","shell.execute_reply.started":"2021-12-12T18:57:40.037605Z","shell.execute_reply":"2021-12-12T18:57:45.849663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 4, 5e-3, 'Volume', 0.65, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:45.853377Z","iopub.execute_input":"2021-12-12T18:57:45.853594Z","iopub.status.idle":"2021-12-12T18:57:48.926540Z","shell.execute_reply.started":"2021-12-12T18:57:45.853567Z","shell.execute_reply":"2021-12-12T18:57:48.925601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 4, 5e-3, 'Close', 0.7, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:48.927766Z","iopub.execute_input":"2021-12-12T18:57:48.927980Z","iopub.status.idle":"2021-12-12T18:57:52.390862Z","shell.execute_reply.started":"2021-12-12T18:57:48.927951Z","shell.execute_reply":"2021-12-12T18:57:52.390024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 5, 5e-4, 'Volume', 0.7, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:52.392154Z","iopub.execute_input":"2021-12-12T18:57:52.392397Z","iopub.status.idle":"2021-12-12T18:57:57.118528Z","shell.execute_reply.started":"2021-12-12T18:57:52.392367Z","shell.execute_reply":"2021-12-12T18:57:57.117639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 5, 5e-3, 'Close', 0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:57:57.119686Z","iopub.execute_input":"2021-12-12T18:57:57.120002Z","iopub.status.idle":"2021-12-12T18:58:03.115686Z","shell.execute_reply.started":"2021-12-12T18:57:57.119973Z","shell.execute_reply":"2021-12-12T18:58:03.114778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 6, 5e-3, 'Volume', 0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:58:03.116946Z","iopub.execute_input":"2021-12-12T18:58:03.117214Z","iopub.status.idle":"2021-12-12T18:58:07.838355Z","shell.execute_reply.started":"2021-12-12T18:58:03.117184Z","shell.execute_reply":"2021-12-12T18:58:07.837355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 6, 5e-3, 'Close', 0.72, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:58:07.839651Z","iopub.execute_input":"2021-12-12T18:58:07.839885Z","iopub.status.idle":"2021-12-12T18:58:14.024207Z","shell.execute_reply.started":"2021-12-12T18:58:07.839857Z","shell.execute_reply":"2021-12-12T18:58:14.023237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 7, 5e-3, 'Volume', 0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T18:58:14.025952Z","iopub.execute_input":"2021-12-12T18:58:14.026510Z","iopub.status.idle":"2021-12-12T18:58:19.058277Z","shell.execute_reply.started":"2021-12-12T18:58:14.026461Z","shell.execute_reply":"2021-12-12T18:58:19.057416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 7, 5e-3, 'Close',0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:00:41.772302Z","iopub.execute_input":"2021-12-12T19:00:41.772697Z","iopub.status.idle":"2021-12-12T19:00:46.815044Z","shell.execute_reply.started":"2021-12-12T19:00:41.772657Z","shell.execute_reply":"2021-12-12T19:00:46.814339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 8, 5e-3, 'Volume', 0.6, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:05:31.586845Z","iopub.execute_input":"2021-12-12T19:05:31.588665Z","iopub.status.idle":"2021-12-12T19:05:38.965152Z","shell.execute_reply.started":"2021-12-12T19:05:31.588596Z","shell.execute_reply":"2021-12-12T19:05:38.964120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 8, 5e-3, 'Close',0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:06:12.991298Z","iopub.execute_input":"2021-12-12T19:06:12.991636Z","iopub.status.idle":"2021-12-12T19:06:18.272213Z","shell.execute_reply.started":"2021-12-12T19:06:12.991600Z","shell.execute_reply":"2021-12-12T19:06:18.271550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 9, 5e-3, 'Volume',0.65, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:15:41.487994Z","iopub.execute_input":"2021-12-12T19:15:41.488350Z","iopub.status.idle":"2021-12-12T19:15:49.278433Z","shell.execute_reply.started":"2021-12-12T19:15:41.488302Z","shell.execute_reply":"2021-12-12T19:15:49.277528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 9, 5e-3, 'Close',0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:15:55.420958Z","iopub.execute_input":"2021-12-12T19:15:55.422162Z","iopub.status.idle":"2021-12-12T19:16:02.036006Z","shell.execute_reply.started":"2021-12-12T19:15:55.422097Z","shell.execute_reply":"2021-12-12T19:16:02.035346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 10, 5e-3, 'Volume',0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:16:45.313583Z","iopub.execute_input":"2021-12-12T19:16:45.313895Z","iopub.status.idle":"2021-12-12T19:16:49.352637Z","shell.execute_reply.started":"2021-12-12T19:16:45.313863Z","shell.execute_reply":"2021-12-12T19:16:49.352008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 11, 5e-3, 'Volume',0.6, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:19:03.112855Z","iopub.execute_input":"2021-12-12T19:19:03.113179Z","iopub.status.idle":"2021-12-12T19:19:07.844304Z","shell.execute_reply.started":"2021-12-12T19:19:03.113145Z","shell.execute_reply":"2021-12-12T19:19:07.843582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 11, 5e-3, 'Close',0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:19:26.175459Z","iopub.execute_input":"2021-12-12T19:19:26.175939Z","iopub.status.idle":"2021-12-12T19:19:32.068824Z","shell.execute_reply.started":"2021-12-12T19:19:26.175885Z","shell.execute_reply":"2021-12-12T19:19:32.067831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 12, 5e-3, 'Volume',0.75, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:28:36.662068Z","iopub.execute_input":"2021-12-12T19:28:36.662790Z","iopub.status.idle":"2021-12-12T19:28:41.588629Z","shell.execute_reply.started":"2021-12-12T19:28:36.662736Z","shell.execute_reply":"2021-12-12T19:28:41.587576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"changepoint_auto(data, 12, 5e-4, 'Close',0.7, False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T19:32:18.426084Z","iopub.execute_input":"2021-12-12T19:32:18.426967Z","iopub.status.idle":"2021-12-12T19:32:24.380643Z","shell.execute_reply.started":"2021-12-12T19:32:18.426896Z","shell.execute_reply":"2021-12-12T19:32:24.379900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TO Do:**\n* cut maker dataset\n* deal with the warnings\n\n**Possible usage in competition:**\n* OOF in line with periods between changepoints\n* categorical features\n* basis for numerical features reflecting trend momentum","metadata":{}}]}