{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introdcution\nPlease upvote👍 if you find this  notebook is useful!\nThis is the introduction notebook for **FFT**.  \nI applyed FFT analysis to time series cryptocurrency data.  \n\nThere are some NAN value in data, which makes FFT does not work.  \nIt is not good, but I tried naive approch which drop the row which has NAN value.  \n\n\nAlso, I tried **Lomb-Scargle periodogram** which can treat those type of data.\n(But this part is pending...)\n\n[💰Wavelet x AI x Stock💰](https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/291909)is related discussion I prepared.","metadata":{}},{"cell_type":"markdown","source":"# Check dataset","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:37:33.051229Z","iopub.execute_input":"2021-12-01T19:37:33.051576Z","iopub.status.idle":"2021-12-01T19:37:33.081059Z","shell.execute_reply.started":"2021-12-01T19:37:33.051487Z","shell.execute_reply":"2021-12-01T19:37:33.080158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_df = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\nasset_df.info()\nasset_df","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:37:33.082581Z","iopub.execute_input":"2021-12-01T19:37:33.082948Z","iopub.status.idle":"2021-12-01T19:37:33.139003Z","shell.execute_reply.started":"2021-12-01T19:37:33.082908Z","shell.execute_reply":"2021-12-01T19:37:33.138113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\n\nsorted_asset_df = asset_df.sort_values('Weight')\n\nx = np.array(sorted_asset_df[\"Asset_Name\"].values)\ny = np.array(sorted_asset_df[\"Weight\"].values)\n\nplt.figure(figsize=(12,8))\nplt.bar(x, y, align=\"center\")\nplt.xticks(rotation=70,fontsize=15)\nplt.title(\"Weight of cryptocurrency\",fontsize=30)\nplt.xlabel(\"Asset Name\",fontsize=20)\nplt.ylabel(\"Weight\",fontsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:37:33.140856Z","iopub.execute_input":"2021-12-01T19:37:33.141693Z","iopub.status.idle":"2021-12-01T19:37:33.488474Z","shell.execute_reply.started":"2021-12-01T19:37:33.141643Z","shell.execute_reply":"2021-12-01T19:37:33.487666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\ntrain_df.info()\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:37:33.489699Z","iopub.execute_input":"2021-12-01T19:37:33.489999Z","iopub.status.idle":"2021-12-01T19:38:34.867303Z","shell.execute_reply.started":"2021-12-01T19:37:33.489965Z","shell.execute_reply":"2021-12-01T19:38:34.866482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Time series plot","metadata":{}},{"cell_type":"code","source":"def plot_time_series(asset_id:int, asset_value:str, start:int = 0, end:int = len(train_df), visualise:bool = True,drop_na = True):\n    \"\"\"Plot time series data from train_df(train.csv)\n\n    Args:\n        asset_id (int): Asset_ID.\n        asset_value (str): Count, Open, High, Low, Close, Volume, VWAP, Target.\n        start(int): start index\n        end(int): end index\n        visualise(bool): If true, the gragh is plot\n        drop_na(bool):\n    \"\"\"\n    \n    asset_name = asset_df[asset_df[\"Asset_ID\"]==asset_id][\"Asset_Name\"].values[0]\n    train_crypro_df = train_df[train_df[\"Asset_ID\"]==asset_id].reset_index(drop=True)\n    \n    if drop_na:\n        train_crypro_df = train_crypro_df.dropna(subset=[asset_value])\n    \n    train_crypro_slice_df = train_crypro_df.iloc[start:end]\n    #train_crypro_df.info()\n    #train_crypro_df.head()\n    t = np.array(list(train_crypro_slice_df.index))\n    y = np.array(train_crypro_slice_df[asset_value].values)\n\n    if visualise:\n        plt.figure(figsize=(12,8))\n        plt.plot(t, y)\n        plt.xticks(fontsize=15)\n        plt.yticks(fontsize=15)\n        plt.title(f\"{asset_value} of {asset_name}\\n(start={start},end={end})\",fontsize=30)\n        plt.xlabel(\"Time\",fontsize=20)\n        plt.ylabel(f\"{asset_value}\",fontsize=20)\n        plt.show()\n    \n    \n    return t, y","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:34.869336Z","iopub.execute_input":"2021-12-01T19:38:34.869656Z","iopub.status.idle":"2021-12-01T19:38:34.880984Z","shell.execute_reply.started":"2021-12-01T19:38:34.869623Z","shell.execute_reply":"2021-12-01T19:38:34.879718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t, y = plot_time_series(asset_id = 8,asset_value = \"Target\",visualise=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:34.882435Z","iopub.execute_input":"2021-12-01T19:38:34.882974Z","iopub.status.idle":"2021-12-01T19:38:36.292326Z","shell.execute_reply.started":"2021-12-01T19:38:34.882915Z","shell.execute_reply":"2021-12-01T19:38:36.291156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t,y = plot_time_series(asset_id = 8,asset_value = \"Target\",start = 0,end = 1000)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:36.293465Z","iopub.execute_input":"2021-12-01T19:38:36.293700Z","iopub.status.idle":"2021-12-01T19:38:36.921444Z","shell.execute_reply.started":"2021-12-01T19:38:36.293670Z","shell.execute_reply":"2021-12-01T19:38:36.920550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t,y = plot_time_series(asset_id = 8,asset_value = \"Close\")","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:36.923401Z","iopub.execute_input":"2021-12-01T19:38:36.923628Z","iopub.status.idle":"2021-12-01T19:38:38.295767Z","shell.execute_reply.started":"2021-12-01T19:38:36.923601Z","shell.execute_reply":"2021-12-01T19:38:38.294840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t,y = plot_time_series(asset_id = 8,asset_value = \"Close\",start = 0,end = 1000)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:38.297397Z","iopub.execute_input":"2021-12-01T19:38:38.297933Z","iopub.status.idle":"2021-12-01T19:38:38.906235Z","shell.execute_reply.started":"2021-12-01T19:38:38.297884Z","shell.execute_reply":"2021-12-01T19:38:38.904966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FFT","metadata":{}},{"cell_type":"markdown","source":"## Example1: sin curve","metadata":{}},{"cell_type":"code","source":"def plot_sin_curve(a:float, f:float, pd:float, end_time:float, sample_num:int,):\n    \"\"\"Generate sin curve and plot\n\n    Args:\n        a (float): Amplitude.\n        f (float): Frequency[Hz].\n        pd(float):phase difference(rad).\n        end_time (float): End time(sec).\n        sample_num(int) sample number.\n    \n    Returns:\n        t(np.ndarray): numpy array of time\n        y(np.ndarray):numpy array of y\n    \"\"\"\n    t = np.linspace(0, end_time, sample_num)\n    y = a * np.sin(2*np.pi*f*t + pd)\n    plt.figure(figsize=(12,8))\n    plt.plot(t,y)\n    plt.grid()\n    plt.show()\n    \n    return t,y","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:38.907778Z","iopub.execute_input":"2021-12-01T19:38:38.908070Z","iopub.status.idle":"2021-12-01T19:38:38.914971Z","shell.execute_reply.started":"2021-12-01T19:38:38.908033Z","shell.execute_reply":"2021-12-01T19:38:38.914062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t,y1 = plot_sin_curve(a = 1.5, f = 50, pd = 0, end_time = 1, sample_num = 1024)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:38.917882Z","iopub.execute_input":"2021-12-01T19:38:38.918159Z","iopub.status.idle":"2021-12-01T19:38:39.157917Z","shell.execute_reply.started":"2021-12-01T19:38:38.918125Z","shell.execute_reply":"2021-12-01T19:38:39.157006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t,y2 = plot_sin_curve(a = 1, f = 120, pd = np.pi/4*0, end_time = 1, sample_num = 1024)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:39.159419Z","iopub.execute_input":"2021-12-01T19:38:39.160228Z","iopub.status.idle":"2021-12-01T19:38:39.457927Z","shell.execute_reply.started":"2021-12-01T19:38:39.160182Z","shell.execute_reply":"2021-12-01T19:38:39.456935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = y1 + y2\n\nplt.figure(figsize=(12,8))\nplt.grid()\nplt.plot(t,y)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:39.459503Z","iopub.execute_input":"2021-12-01T19:38:39.459800Z","iopub.status.idle":"2021-12-01T19:38:39.747082Z","shell.execute_reply.started":"2021-12-01T19:38:39.459736Z","shell.execute_reply":"2021-12-01T19:38:39.746356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_fft(y:np.ndarray, t:np.ndarray, fs:int, sample_num:int, value_name:str):\n    \"\"\"Generate fft and plot\n\n    Args:\n        y (np.ndarray): time series value.\n        t (np.ndarray): time.\n        fs(int):sampling frequency.\n        sample_num(int): sample number.\n        value_name(str): name of value\n    \"\"\"\n    \n    sp = np.fft.fft(y)\n    freq = np.fft.fftfreq(t.shape[-1], d=1.0/fs)\n\n#     fig, ax = plt.subplots(nrows=3, sharex=True, figsize=(12,12))\n#     ax[0].plot(sp.real, label=\"Real part\")\n#     ax[0].legend()\n#     ax[1].plot(sp.imag, label=\"Imaginary part\")\n#     ax[1].legend()\n#     ax[2].plot(freq, label=\"Frequency\")\n#     ax[2].legend()\n#     ax[2].set_xlabel(\"Number of data\")\n\n#     plt.grid()\n#     plt.show()\n    \n    Amp = np.abs(sp/(sample_num/2)) # Amplitude\n\n    fig, ax = plt.subplots(figsize=(12,8))\n    ax.plot(freq[1:int(sample_num/2)], Amp[1:int(sample_num/2)])\n    plt.title(f\"FFT of {value_name}\",fontsize=30)\n    ax.set_xlabel(\"Freqency [Hz]\")\n    ax.set_ylabel(\"Amplitude\")\n    ax.grid()\n    #plt.xlim(0,5)#Change range according to frequency of wave\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:39.748188Z","iopub.execute_input":"2021-12-01T19:38:39.748548Z","iopub.status.idle":"2021-12-01T19:38:39.757134Z","shell.execute_reply.started":"2021-12-01T19:38:39.748518Z","shell.execute_reply":"2021-12-01T19:38:39.756176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_fft(y=y,t=t,fs=1000,sample_num=1024,value_name=\"sin wave\")","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:39.758178Z","iopub.execute_input":"2021-12-01T19:38:39.758399Z","iopub.status.idle":"2021-12-01T19:38:40.011198Z","shell.execute_reply.started":"2021-12-01T19:38:39.758364Z","shell.execute_reply":"2021-12-01T19:38:40.010223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Example2: White noize","metadata":{}},{"cell_type":"code","source":"import numpy as np\ndlen = 1024 #ノイズデータのデータ長\nmean = 0.0  #ノイズの平均値\nstd  = 1.0  #ノイズの分散\n\ny = np.random.normal(mean,std,dlen)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:40.012361Z","iopub.execute_input":"2021-12-01T19:38:40.012616Z","iopub.status.idle":"2021-12-01T19:38:40.018539Z","shell.execute_reply.started":"2021-12-01T19:38:40.012585Z","shell.execute_reply":"2021-12-01T19:38:40.017413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_white_noize(mean:float, std:float, end_time:float, sample_num:int):\n    \"\"\"Generate sin curve and plot\n\n    Args:\n        mean (float): average.\n        std(float):standard deviation.\n        end_time (float): End time(sec).\n        sample_num(int) sample number.\n    \n    Returns:\n        t(np.ndarray): numpy array of time\n        y(np.ndarray):numpy array of y\n    \"\"\"\n    t = np.linspace(0, end_time, sample_num)\n    y = np.random.normal(mean,std,sample_num)\n    plt.figure(figsize=(12,8))\n    plt.plot(t,y)\n    plt.grid()\n    plt.show()\n    \n    return t,y","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:40.019925Z","iopub.execute_input":"2021-12-01T19:38:40.020169Z","iopub.status.idle":"2021-12-01T19:38:40.033677Z","shell.execute_reply.started":"2021-12-01T19:38:40.020138Z","shell.execute_reply":"2021-12-01T19:38:40.032907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t, y = plot_white_noize(mean=0, std=1.0, end_time=10, sample_num=1024)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:40.034936Z","iopub.execute_input":"2021-12-01T19:38:40.035736Z","iopub.status.idle":"2021-12-01T19:38:40.418830Z","shell.execute_reply.started":"2021-12-01T19:38:40.035694Z","shell.execute_reply":"2021-12-01T19:38:40.417869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_fft(y=y,t=t,fs=1000,sample_num=1024,value_name=\"White noize\")","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:40.420214Z","iopub.execute_input":"2021-12-01T19:38:40.420953Z","iopub.status.idle":"2021-12-01T19:38:40.693034Z","shell.execute_reply.started":"2021-12-01T19:38:40.420903Z","shell.execute_reply":"2021-12-01T19:38:40.692017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FFT for cryptocurrency","metadata":{}},{"cell_type":"code","source":"for asset_id in asset_df[\"Asset_ID\"].values:\n    value_name = asset_df[asset_df[\"Asset_ID\"]==asset_id][\"Asset_Name\"].values[0]\n    t, y = plot_time_series(asset_id = asset_id, asset_value = \"Target\",start = 0, end = 1024, visualise=False)\n    plot_fft(y=y,t=t,fs=1000,sample_num=1024,value_name=value_name)","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:40.694501Z","iopub.execute_input":"2021-12-01T19:38:40.694828Z","iopub.status.idle":"2021-12-01T19:38:49.140828Z","shell.execute_reply.started":"2021-12-01T19:38:40.694790Z","shell.execute_reply":"2021-12-01T19:38:49.139842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lomb-Scargle periodogram","metadata":{}},{"cell_type":"markdown","source":"For details of Lomb-Scargle periodogram, please check below reference.\n\n**Caution**  \nFor now, this does not work properly, if \"drop_na = False\", it does not output any...  \nSo it is set as True.","metadata":{}},{"cell_type":"code","source":"import scipy.signal as signal\n\nasset_id = 8\nvalue_name = asset_df[asset_df[\"Asset_ID\"]==asset_id][\"Asset_Name\"].values[0]\nt, y = plot_time_series(asset_id = asset_id,asset_value = \"Target\",start = 0, end = 1024, visualise=False,drop_na = True )\n\nf = np.linspace(0.01, 100, 10000)\npgram = signal.lombscargle(t, y, f, normalize=True)\nplt.plot(f, pgram)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-01T19:38:49.141914Z","iopub.execute_input":"2021-12-01T19:38:49.142144Z","iopub.status.idle":"2021-12-01T19:38:51.254547Z","shell.execute_reply.started":"2021-12-01T19:38:49.142117Z","shell.execute_reply":"2021-12-01T19:38:51.253642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reference\n1. [numpy.fft.fft](https://numpy.org/doc/stable/reference/generated/numpy.fft.fft.html)\n2. [Spectral Analysis of Nonuniformly Sampled Signals](https://www.mathworks.com/help/signal/ug/spectral-analysis-of-nonuniformly-sampled-signals.html)\n3. [scipy.signal.lombscargle](https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.lombscargle.html)","metadata":{}}]}