{"cells":[{"metadata":{"_uuid":"b8644c308dc0f624968478fe20cf67d27bb83826"},"cell_type":"markdown","source":"This kernel goes with a medium posts I write, on how to *`Tackle`* problems faced in DataScience,published in Towards Data Science. Here are all the posts:\n1. [How to handle BigData Files on Low Memory](https://towardsdatascience.com/how-to-learn-from-bigdata-files-on-low-memory-incremental-learning-d377282d38ff)"},{"metadata":{"_uuid":"944847297586c25f5491664051d7b95cf762a00f"},"cell_type":"markdown","source":"# Initial"},{"metadata":{"trusted":true,"_uuid":"72ecd515b533ce9614edec0919437be00304e231"},"cell_type":"code","source":"#!pip -q install --upgrade --ignore-installed numpy pandas scipy sklearn","execution_count":null,"outputs":[]},{"metadata":{"id":"bR_7nQDHtt70","colab_type":"code","colab":{},"trusted":true,"_uuid":"cd9012ecf4e031054689619f2ab802bc7b8d7f66"},"cell_type":"code","source":"#!pip -q install catboost\n#!pip -q install lightgbm","execution_count":null,"outputs":[]},{"metadata":{"id":"P-IrGI84NjyG","colab_type":"code","colab":{},"trusted":true,"_uuid":"8d75250e2f2285458e2a65213c34fe7b018de43b"},"cell_type":"code","source":"#!pip -q install \"dask[complete]\"\n#!pip -q install \"dask-ml[complete]\"","execution_count":null,"outputs":[]},{"metadata":{"id":"JxR5tH77_BWS","colab_type":"code","colab":{},"trusted":true,"_uuid":"3a76192fa9b963276dcc9b062815df11591042ca"},"cell_type":"code","source":"## https://stackoverflow.com/questions/49853303/how-to-install-pydot-graphviz-on-google-colab?rq=1\n#!pip -q install graphviz \n#!apt-get install graphviz -qq\n#!pip -q install pydot","execution_count":null,"outputs":[]},{"metadata":{"id":"HN9y6AADMe6E","colab_type":"code","colab":{},"trusted":true,"_uuid":"08aa0eb5fc1982f6775249a5a4a61d247f465975"},"cell_type":"code","source":"# After this restart your kernel using Ctrl+M+. to reset all loaded libraries to latest ones, that we just installed.","execution_count":null,"outputs":[]},{"metadata":{"id":"32g2_Q82nWAi","colab_type":"text","_uuid":"0cfa9abfa41468f2d44d13252feb4dea64d23ba2"},"cell_type":"markdown","source":"# Import"},{"metadata":{"id":"Uu1Dzz0nnX8C","colab_type":"code","colab":{},"trusted":true,"_uuid":"4d614b581992504c7570a6a376c5d1bff4851ffe"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom multiprocessing import Pool\nfrom pandas.io.json import json_normalize\nfrom sklearn.pipeline import make_pipeline, Pipeline\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import RobustScaler, LabelEncoder\nfrom threading import Thread as trd\nimport queue\nimport json\nimport gc\ngc.enable()","execution_count":null,"outputs":[]},{"metadata":{"id":"5QU_6EBwdXjR","colab_type":"text","_uuid":"ff2ab616bd643a540ce2390144022bbf68b7798d"},"cell_type":"markdown","source":"# Incremental Learning"},{"metadata":{"id":"vhC1BguWtFcz","colab_type":"text","_uuid":"cf3ddf9ba00ff8bcb867ca281c212cd302482578"},"cell_type":"markdown","source":"### Data Exploration:"},{"metadata":{"id":"mHy_qxCjmPuU","colab_type":"code","colab":{},"trusted":true,"_uuid":"630d638e1a3984128118e0b4dd9993e64d4c5afc"},"cell_type":"code","source":"PATH = \"../input/ga-customer-revenue-prediction/\"","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"AqSBBruLtL9d","outputId":"30b315c7-f6d3-4c45-934c-e2e7fb631e38","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true,"_uuid":"f6b89c2232b45c9059e5766ead6e8c7aee821cff"},"cell_type":"code","source":"part = pd.read_csv(PATH+\"train_v2.csv\",dtype={\"fullVisitorId\":\"str\", \"visitId\":\"str\"}, nrows=10)\npart.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"gWiN2k8WuGDM","colab_type":"code","outputId":"559ea0e3-b25a-4358-b81a-8a4c57e9ae7e","colab":{"base_uri":"https://localhost:8080/","height":106},"trusted":true,"_uuid":"c75f79a10b2eb1d78eef05ff4d187b9730430b12"},"cell_type":"code","source":"columns = part.columns\npart.head(2)","execution_count":null,"outputs":[]},{"metadata":{"id":"CsvUwefIu-xU","colab_type":"code","colab":{},"trusted":true,"_uuid":"f6458d8171e4ea8e8e0511f84471da3025af3150"},"cell_type":"code","source":"# From above output you will have a basic understanding of type of columns. Divide them so you\n# can use same function on similar columns (for exploration etc).\nid_columns = [\"fullVisitorId\", \"visitId\", \"visitStartTime\"]\nnum_columns = [\"visitNumber\"]\nobj_columns = [\"channelGrouping\", \"socialEngagementType\", \"date\"]\ndict_columns = [ \"device\", \"geoNetwork\", \"totals\", \"trafficSource\"]\ncomplex_columns = [\"customDimensions\", \"hits\"]","execution_count":null,"outputs":[]},{"metadata":{"id":"IyL3EFXFAJyu","colab_type":"code","outputId":"ea58f036-972d-4a7a-a6c2-27050f8dceb5","colab":{"base_uri":"https://localhost:8080/","height":195},"trusted":true,"_uuid":"e52f303ceb5bea605a1236ab034fdb7ce7112993","scrolled":true},"cell_type":"code","source":"col = \"device\"\ndf = pd.read_csv(PATH+\"train_v2.csv\", usecols = [3], converters={col: json.loads})\n\ncolumn_as_df = json_normalize(df[col])\ncolumn_as_df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"GeW_nDzpB4An","colab_type":"code","colab":{},"trusted":true,"_uuid":"762b15153f937c70e3b0b6053c4d770682add1b2"},"cell_type":"code","source":"# Explore this column. Check if you want to keep it or not.\n# If you want to scale this column or LableEncode this column, \n# do it and maintain a dictionary with all columns as keys and\n# corresponding Scalar or LabelEncoder or both as values.","execution_count":null,"outputs":[]},{"metadata":{"id":"XE4ZA4q5l1q0","colab_type":"code","outputId":"06cb9816-3c67-4a40-d1e0-0ae74c1afe8f","colab":{"base_uri":"https://localhost:8080/","height":67},"trusted":true,"_uuid":"4f979aeaae6b3d8198982aecd7e828a578703f79","scrolled":false},"cell_type":"code","source":"# Calculate Standard Deviation or check number of uniques, check if they are constant...\ncolumn_as_df.nunique()","execution_count":null,"outputs":[]},{"metadata":{"id":"96nFdSBatrEc","colab_type":"code","outputId":"b6b2e9b0-0cfa-4dc0-f17b-53435398c74f","colab":{"base_uri":"https://localhost:8080/","height":364},"trusted":true,"_uuid":"5c6f859b710066fe002ea21f730fc2f9f1233e24"},"cell_type":"code","source":"drop_cols = [col for col in column_as_df.columns if col not in ['browser', 'deviceCategory', 'isMobile', 'operatingSystem']]\ncolumn_as_df.drop(drop_cols, axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26bb3ae4e380f0c11dcc1dec6fe2428a4a2654b0"},"cell_type":"code","source":"column_as_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7470c6e5bf540092b4039fcb430159d66c72c80a"},"cell_type":"code","source":"final_selected_columns = {\n    \"device\": [\"browser\", \"deviceCategory\", \"isMobile\", \"operatingSystem\"],\n    \"geoNetwork\":[\"city\", \"continent\", \"country\", \"metro\", \"networkDomain\", \"region\", \"subContinent\"],\n    \"totals\":[\"transactions\", \"sessionQualityDim\", \"hits\", \"pageviews\", \"totalTransactionRevenue\", \"timeOnSite\", \"transactionRevenue\"],\n    \"trafficSource\":[\"adContent\", \"adwordsClickInfo.adNetworkType\", \"adwordsClickInfo.page\", \"adwordsClickInfo.slot\",\n                     \"campaign\", \"keyword\",\t\"medium\", \"referralPath\", \"source\"],\n    \"customDimensions\": [\"customDimensions\"],\n    \"channelGrouping\": [\"channelGrouping\"],\n    \"date\":[\"date\"],\n    \"visitNumber\":[\"visitNumber\"],\n    \"visitStartTime\":[\"visitStartTime\"]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"5d85008e839d63a0121597319995da411f273993"},"cell_type":"code","source":"columns_selected = []\nfor key in final_selected_columns.keys():\n    if len(final_selected_columns[key]) == 1:\n        columns_selected.append(key)\n    else:\n        for sub_col in final_selected_columns[key]:\n            columns_selected.append(f\"{key}.{sub_col}\")\ncolumns_selected","execution_count":null,"outputs":[]},{"metadata":{"id":"Qb3ZWBRlosY0","colab_type":"code","colab":{},"trusted":true,"_uuid":"d0b4f217b14b2c49ad3a19156abefa08997d8532"},"cell_type":"code","source":"prep_pipeline = {col: [] for col in columns_selected}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dca34a740d4842ecf0e2f0db04ebbffc38aebd7"},"cell_type":"code","source":"for col_ in [\"browser\", \"deviceCategory\", \"operatingSystem\"]:\n    lb = LabelEncoder()\n    column_as_df[col_] = lb.fit_transform(column_as_df[col_].values)\n    prep_pipeline[f\"device.{col_}\"].append(lb)","execution_count":null,"outputs":[]},{"metadata":{"id":"44Gu-D6tpqbP","colab_type":"code","colab":{},"trusted":true,"_uuid":"30f3ccca8a30a775b6a36d1896d2b12a1dea6c42"},"cell_type":"code","source":"for col_ in [\"browser\", \"deviceCategory\", \"operatingSystem\"]:\n    rsc = RobustScaler()\n    column_as_df[col_] = rsc.fit_transform(column_as_df[col_].values.reshape(-1, 1)).reshape(1, -1)[0]\n    prep_pipeline[f\"device.{col_}\"].append(rsc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95eeeb37423744796fa625a21419e6ea11dea590"},"cell_type":"code","source":"column_as_df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"owZ7c_c7zl1-","colab_type":"code","colab":{},"trusted":true,"_uuid":"b2a6e105fdfec9df572ce087b4afb8ac02752cc9"},"cell_type":"code","source":"import pickle\nwith open(\"pipeline.pickle\", \"wb\") as fle:\n    pickle.dump(prep_pipeline, fle)\n!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36e696127d918a9eef31098b39f0c3113eafc2b8"},"cell_type":"code","source":"del df, column_as_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"2A4Rvd5vwaaf","colab_type":"text","_uuid":"cb37de264e14c0ceef9834bee917170230852a6c"},"cell_type":"markdown","source":"**Note:**\n\nIf even one column of dataset is larger than your memory, then you can open that column incrementally (by using `chunksize` parameter in `pd.read_csv`), applying some transformation (maybe keeping it in a `np.float32` if its a number or maybe keeping only relevant part of a string and throwing all other away.)\n\nBut if that column is not fitting into your memory, you can use `Dask`. (see last section)"},{"metadata":{"id":"iNVpOm4Vhwz_","colab_type":"text","_uuid":"4ada38e89b566973d840c226601afda9dad96de4"},"cell_type":"markdown","source":"### Preprocessing method:"},{"metadata":{"trusted":true,"_uuid":"5732d0d3129a33cb8f7d6d5e9b375f3b4ebd1ff0"},"cell_type":"code","source":"PATH2 = \"../input/googleanalbigfile/\"\n!ls {PATH2}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89d70aed1cfb34542e694d93436eb2501b6c18d2"},"cell_type":"code","source":"with open(PATH2+\"preprocessing_pipeline (1).pickle\", \"rb\") as fle:\n    preprocessing_pipeline = pickle.load(fle)","execution_count":null,"outputs":[]},{"metadata":{"id":"5Cc0jrK9GCHC","colab_type":"code","colab":{},"trusted":true,"_uuid":"7b08351ef074a4a7b58aea0cd5da3408580cb989"},"cell_type":"code","source":"final_selected_columns = {\n    \"device\": [\"browser\", \"deviceCategory\", \"isMobile\", \"operatingSystem\"],\n    \"geoNetwork\":[\"city\", \"continent\", \"country\", \"metro\", \"networkDomain\", \"region\", \"subContinent\"],\n    \"totals\":[\"transactions\", \"sessionQualityDim\", \"hits\", \"pageviews\", \"totalTransactionRevenue\", \"timeOnSite\", \"transactionRevenue\"],\n    \"trafficSource\":[\"adContent\", \"adwordsClickInfo.adNetworkType\", \"adwordsClickInfo.page\", \"adwordsClickInfo.slot\",\n                     \"campaign\", \"keyword\",\t\"medium\", \"referralPath\", \"source\"],\n    \"customDimensions\": [\"customDimensions\"],\n    \"channelGrouping\": [\"channelGrouping\"],\n    \"date\":[\"date\"],\n    \"visitNumber\":[\"visitNumber\"],\n    \"visitStartTime\":[\"visitStartTime\"]\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae8d3b633901726356fed106a2c2d5e5d0486730"},"cell_type":"markdown","source":"These functions are taken from [here](https://www.kaggle.com/prashantkikani/teach-lightgbm-to-sum-predictions-fe)."},{"metadata":{"trusted":true,"_uuid":"73f6c0f049c340a937841af5e2bc953507e8b4e2"},"cell_type":"code","source":"def browser_mapping(x):\n    browsers = ['chrome','safari','firefox','explorer','edge','opera','coc coc','maxthon','iron']\n    for br in browsers: \n      if br in x.lower(): return br\n    if  ('android' in x) or ('samsung' in x) or ('mini' in x) or ('iphone' in x) or ('in-app' in x) or ('playstation' in x): return 'mobile browser'\n    elif  ('mozilla' in x) or ('chrome' in x) or ('blackberry' in x) or ('nokia' in x) or ('browser' in x) or ('amazon' in x): return 'mobile browser'\n    elif  ('lunascape' in x) or ('netscape' in x) or ('blackberry' in x) or ('konqueror' in x) or ('puffin' in x) or ('amazon' in x): return 'mobile browser'\n    elif '(not set)' in x: return '(not set)'\n    else: return 'others'\n\ndef adcontents_mapping(x):\n    if 'google' in x: return 'google'\n    elif ('placement' in x) | ('placememnt' in x): return 'placement'\n    elif '(not set)' in x: return '(not set)'\n    elif 'na' == x: return 'na'\n    elif 'ad' in x: return 'ad'\n    else: return 'others'\n    \ndef keyword_mapping(x):\n  if '(not provided)' in x: return '(not provided)'\n  elif 'na' == x: return 'na'\n  elif 'google' in x: return 'google'\n  elif 'shart' in x: return 't-shirt'\n  elif 'sticker' in x: return 'sticker'\n  elif 'lav' in x: return 'lava-lamp'\n  elif 'pen' in x: return 'pen'\n  elif 'merc' in x: return 'merchandise'\n  elif 'bag' in x: return 'bagpack'\n  elif 'laptop' in x: return 'laptop'\n  elif 'onesie' in x: return 'onesie'\n  elif 'sunglass' in x: return 'sunglasses'\n  elif 'cup' in x: return 'cup'\n  elif 'cap' in x: return 'cap'\n  elif 'arrel' in x: return 'apparel'\n  elif 'arel' in x: return 'apparel'\n  elif 'shirt' in x: return 't-shirt'\n  elif 'jac' in x: return 'jacket'\n  elif 'bank' in x: return 'powerbank'\n  elif 'dre' in x: return 'dress'\n  elif 'todd' in x: return 'toddler'\n  elif 'bic' in x: return 'bicycle'\n  elif 'earp' in x: return 'earphone'\n  elif 'head' in x: return 'headphones'\n  elif 'virt' in x: return 'virtual-reality'\n  elif 'reali' in x: return 'virtual-reality'\n  elif 'bott' in x: return 'bottle'\n  elif 'book' in x: return 'notebook'\n  elif 'men' in x: return 'men'\n  elif 'women' in x: return 'women'\n  elif 'wom' in x: return 'women'\n  elif 'goo' in x: return 'google'\n  elif 'gle' in x: return 'google'\n  elif 'you' in x: return 'youtube'\n  elif 'yu' in x: return 'youtube'\n  elif 'yi' in x: return 'youtube'\n  elif 'ube' in x: return 'youtube'\n  elif 'tb' in x: return 'youtube'\n  else: return 'others'\n  \ndef referralPath_mapping(x):\n  if 'intl' in x: return 'intl'\n  elif 'yt' in x: return 'yt'\n  elif 'golang' in x: return 'golang'\n  elif 'document' in x: return 'document'\n  elif '/r/' in x: return 'reddit'\n  elif 'google' in x: return 'google'\n  elif '/ads' in x: return 'ads'\n  elif 'ads/' in x: return 'ads'\n  elif 'adsense/' in x: return 'ads'\n  elif 'mail' in x: return 'mail'\n  elif 'hangouts' in x: return 'hangouts'\n  elif 'iphone' in x: return 'iphone'\n  elif 'webapps/' in x: return 'webapp'\n  elif 'maps/' in x: return 'maps'\n  elif 'webmasters/' in x: return 'webapp'\n  elif 'android/' in x: return 'android'\n  elif 'websearch/' in x: return 'websearch'\n  elif 'youtube/' in x: return 'youtube'\n  elif 'spreadsheets/' in x: return 'spreadsheets'\n  elif 'presentation/' in x: return 'presentation'\n  elif 'chromebook/' in x: return 'chromebook'\n  elif 'bookmarks/' in x: return 'bookmark'\n  elif 'drive/' in x: return 'drive'\n  elif 'baiduid' in x: return 'baidu'\n  elif 'calendar/' in x: return 'calendar'\n  elif 'pin/' in x: return 'pinterest'\n  elif 'entry/' in x: return 'entry'\n  elif 'chrome/' in x: return 'chrome'\n  elif '.html' in x: return 'html'\n  elif '.pdf' in x: return 'pdf'\n  elif 'wiki/' in x: return 'wiki'\n  elif 'appserve/' in x: return 'webapp'\n  elif 'web/' in x: return 'web'\n  elif 'edit/' in x: return 'edit'\n  elif 'feed/' in x: return 'feed'\n  elif 'feedback' in x: return 'feedback'\n  elif 'user/' in x: return 'user'\n  elif '.htm' in x: return 'htm'\n  elif '.php' in x: return 'php'\n  elif 'class_section' in x: return 'class_section'\n  elif 'messages/' in x: return 'message'\n  elif 'na' == x: return 'na'\n  else: return 'others'\n  \ndef source_mapping(x):\n    if 'google' in x:return 'google'\n    elif 'youtube' in x:return 'youtube'\n    elif 'yahoo' in x:return 'yahoo'\n    elif 'facebook' in x:return 'facebook'\n    elif 'reddit' in x:return 'reddit'\n    elif 'bing' in x:return 'bing'\n    elif 'quora' in x:return 'quora'\n    elif 'outlook' in x:return 'outlook'\n    elif 'linkedin' in x:return 'linkedin'\n    elif 'pinterest' in x:return 'pinterest'\n    elif 'ask' in x:return 'ask'\n    elif 'siliconvalley' in x:return 'siliconvalley'\n    elif 'lunametrics' in x:return 'lunametrics'\n    elif 'amazon' in x:return 'amazon'\n    elif 'mysearch' in x:return 'mysearch'\n    elif 'qiita' in x:return 'qiita'\n    elif 'messenger' in x:return 'messenger'\n    elif 'twitter' in x:return 'twitter'\n    elif 't.co' in x:return 't.co'\n    elif 'vk.com' in x:return 'vk.com'\n    elif 'search' in x:return 'search'\n    elif 'edu' in x:return 'edu'\n    elif 'mail' in x:return 'mail'\n    elif 'ad' in x:return 'ad'\n    elif 'golang' in x:return 'golang'\n    elif 'direct' in x:return 'direct'\n    elif 'dealspotr' in x:return 'dealspotr'\n    elif 'sashihara' in x:return 'sashihara'\n    elif 'phandroid' in x:return 'phandroid'\n    elif 'baidu' in x:return 'baidu'\n    elif 'mdn' in x:return 'mdn'\n    elif 'duckduckgo' in x:return 'duckduckgo'\n    elif 'seroundtable' in x:return 'seroundtable'\n    elif 'metrics' in x:return 'metrics'\n    elif 'sogou' in x:return 'sogou'\n    elif 'businessinsider' in x:return 'businessinsider'\n    elif 'github' in x:return 'github'\n    elif 'gophergala' in x:return 'gophergala'\n    elif 'yandex' in x:return 'yandex'\n    elif 'msn' in x:return 'msn'\n    elif 'dfa' in x:return 'dfa'\n    elif 'feedly' in x:return 'feedly'\n    elif 'arstechnica' in x:return 'arstechnica'\n    elif 'squishable' in x:return 'squishable'\n    elif 'flipboard' in x:return 'flipboard'\n    elif 't-online.de' in x:return 't-online.de'\n    elif 'sm.cn' in x:return 'sm.cn'\n    elif 'wow' in x:return 'wow'\n    elif 'baidu' in x:return 'baidu'\n    elif 'partners' in x:return 'partners'\n    elif '(not set)' in x: return \"(not set)\"\n    elif 'nan' in x: return 'na'\n    else: return 'others'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2591327af40b95ce0dd5e636e8831b26df083cb2"},"cell_type":"code","source":"dict_columns = [ \"device\", \"geoNetwork\", \"totals\", \"trafficSource\"]\nid_columns = [\"fullVisitorId\", \"visitId\"]\nnum_columns = ['totals.transactions', 'totals.sessionQualityDim', 'totals.hits',\n               'totals.pageviews', 'totals.totalTransactionRevenue', 'totals.timeOnSite', \n              'trafficSource.adwordsClickInfo.page', 'totals.transactionRevenue', 'visitNumber']\nprocess_columns = {'device.browser': browser_mapping, \n                   'trafficSource.adContent': adcontents_mapping, \n                   'trafficSource.source': source_mapping,\n                   'trafficSource.keyword': keyword_mapping, \n                   'trafficSource.referralPath': referralPath_mapping}\ndrop_columns = [\"hits\", \"socialEngagementType\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59cef50f22623cd50b55615ce0e44b548162021d"},"cell_type":"code","source":"all_columns = columns_selected","execution_count":null,"outputs":[]},{"metadata":{"id":"xC9huFOkhwh_","colab_type":"code","colab":{},"trusted":true,"_uuid":"b12f8d6b670f15e1a4740c39562b6c3d43d291dd"},"cell_type":"code","source":"# Now that you have a dictionary which has all the columns you need as keys\n# and corresponding methods you want to apply to that column sequentially,\n# you can make a preprocesing method which will be used to clean data at\n# every incremental step.\n\ndef preprocess(df):\n  df.reset_index(drop=True, inplace=True)\n  \n  for col_ in drop_columns:\n    if col_ in df.columns: df = df.drop([col_], axis=1)\n  \n  for col in dict_columns:\n    col_df = json_normalize(df[col])\n    col_df.columns = [f\"{col}.{subcolumn}\" for subcolumn in col_df.columns]\n    \n    to_drop = [temp for temp in col_df.columns if temp not in all_columns]\n    col_df = col_df.drop(to_drop, axis=1)\n    \n    df = df.drop([col], axis=1).merge(col_df, right_index=True, left_index=True)\n    \n    for temp_col in col_df.columns:\n      if temp_col in num_columns: df[temp_col] = df[temp_col].fillna(0.0).astype(float)\n      elif temp_col == \"device.isMobile\": df[temp_col] = df[temp_col].astype(bool)\n      else: df[temp_col] = df[temp_col].fillna('NA').astype(str)\n        \n      if temp_col in process_columns.keys(): df[temp_col] = df[temp_col].map(lambda x: process_columns[temp_col](str(x).lower())).astype('str')        \n      \n      if len(preprocessing_pipeline[temp_col]) >= 4:\n        for i in range(4):\n          if preprocessing_pipeline[temp_col][i] is None: continue\n          elif i == 0: \n            lb = preprocessing_pipeline[temp_col][i]\n            df[temp_col] = lb.transform(df[temp_col])\n          elif i == 1:\n            df[temp_col] = np.log1p(df[temp_col])\n          elif i == 3:\n            pass\n            #rsc = preprocessing_pipeline[temp_col][i]\n            #df[temp_col] = rsc.transform(df[temp_col].values.reshape(-1, 1)).reshape(1, -1)[0]\n      elif len(preprocessing_pipeline[temp_col]) == 3:\n        for i in range(3):\n          if preprocessing_pipeline[temp_col][i] is None: continue\n          elif i == 0: \n            lb = preprocessing_pipeline[temp_col][i]\n            df[temp_col] = lb.transform(df[temp_col])\n          elif i == 2:\n            pass\n            #rsc = preprocessing_pipeline[temp_col][i]\n            #df[temp_col] = rsc.transform(df[temp_col].values.reshape(-1, 1)).reshape(1, -1)[0]\n            \n  df['visitNumber'] = df['visitNumber'].fillna(0.0).astype(float)\n  df['channelGrouping'] = df['channelGrouping'].fillna('NA').astype(str)\n  df['customDimensions'] = df['customDimensions'].fillna('NA').astype(str)\n  \n  for col_ in [\"visitNumber\", \"customDimensions\", \"channelGrouping\"]:\n    for i in range(3):\n      if preprocessing_pipeline[col_][i] is None: continue\n      elif i==0:\n        lb = preprocessing_pipeline[col_][i]\n        df[col_] = lb.transform(df[col_])\n      elif i == 2:\n        pass\n        #rsc = preprocessing_pipeline[col_][i]\n        #df[col_] = rsc.transform(df[col_].values.reshape(-1, 1)).reshape(1, -1)[0]\n  \n  df['date_new'] = pd.to_datetime(df['visitStartTime'], unit='s')\n  df.drop(['visitStartTime'], axis=1, inplace=True)\n  df['sess_date_dow'] = df['date_new'].dt.dayofweek\n  df['sess_date_hours'] = df['date_new'].dt.hour\n  df['sess_month'] = df['date_new'].dt.month\n  df['sess_year'] = df['date_new'].dt.year\n  \n  for col_ in ['totals.transactions', 'totals.totalTransactionRevenue']:\n    if col_ in df.columns: df = df.drop([col_], axis=1)\n  \n  return df","execution_count":null,"outputs":[]},{"metadata":{"id":"Cs1Id8TeEQ-E","colab_type":"text","_uuid":"53084e008806602ef6b332d098a65150138a2c85"},"cell_type":"markdown","source":"**Note:**\n\nOne thing to notice here is that we fitted methods (like LabelEncoder's, Scalars's etc.) during exploration to **whole** data column and we will use that to transform data at every incremental step here. Because, in each batch, there might be some data missing and if we had used different LabelEncoder's, Scaler's etc. for each batch, these methods wouldn't have given same result for same category (say). That's why to be on safe side, we already have fitted to whole columns during exploration."},{"metadata":{"id":"yJiOYauYePQO","colab_type":"text","_uuid":"4694d647c2a1a289316221b918cdabda486ac572"},"cell_type":"markdown","source":"### Method 1: Using Pandas"},{"metadata":{"id":"PEgsCVSZuefl","colab_type":"code","colab":{},"trusted":true,"_uuid":"74652ff8adc10f8690790f5cf2e60f3853ab590e"},"cell_type":"code","source":"import lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb # CatBoost is currently making its incremental learner: https://github.com/catboost/catboost/issues/464","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbc9d73a16ee79873d73690005e9c5c3780cf98c"},"cell_type":"code","source":"dict_columns = [ \"device\", \"geoNetwork\", \"totals\", \"trafficSource\"]\nid_columns = [\"fullVisitorId\", \"visitId\"]","execution_count":null,"outputs":[]},{"metadata":{"id":"u_iS928pdouO","colab_type":"code","colab":{},"trusted":true,"_uuid":"1c227322e286c65f6d2cc4f8e2e11af33b5f0cf5"},"cell_type":"code","source":"incremental_dataframe = pd.read_csv(PATH + \"train_v2.csv\",\n                                   dtype={\"fullVisitorId\":\"str\", \"visitId\":\"str\"},\n                                   converters = {col_: json.loads for col_ in dict_columns},\n                                   chunksize=100000) # Number of lines to read.\n# This method will return a sequential file reader reading 'chunksize' lines every time.\n# To read file from starting again, you will have to call this method again.","execution_count":null,"outputs":[]},{"metadata":{"id":"orP3rs6c8lVY","colab_type":"code","colab":{},"trusted":true,"_uuid":"a6c81e518a503ea3d5666024574d076df24dadf5"},"cell_type":"code","source":"lgb_params = {\n  \"objective\" : \"regression\",\n  \"n_estimators\":3000,\n  \"max_depth\": 4,\n  \"metric\" : \"rmse\",\n  \"learning_rate\" : 0.01,\n  'subsample':.8,\n  'colsample_bytree':.9\n}\n# First three are necessary for incremental learning.\nxgb_params = {\n  'update':'refresh',\n  'process_type': 'update',\n  'refresh_leaf': True,\n  'silent': True,\n  }","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"517d321f6e7016d601b71c134c12b5e4446a3c48"},"cell_type":"markdown","source":"This function is taken from [here](https://www.kaggle.com/ogrellier/i-have-seen-the-future)."},{"metadata":{"trusted":true,"_uuid":"1f0d8f4886e5e49d8283c0e61f174f442b29306d"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a0e1320d83a0f166a20f51c48597cc79f2a63f3"},"cell_type":"code","source":"drop_cols =['date', 'visitId', 'date_new', 'fullVisitorId', 'totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"id":"vLA0b2cNe1Ph","colab_type":"code","outputId":"4b432342-af02-4b03-bcdd-165bc345bd10","colab":{"base_uri":"https://localhost:8080/","height":1042},"trusted":true,"_uuid":"7d19388e1b808551e506b307d3646e321a631615","scrolled":true},"cell_type":"code","source":"# For saving regressor for next use.\nlgb_estimator = None\n\ni=0\nprev=None\nfor df in incremental_dataframe:\n  print(i)\n  df = preprocess(df)\n  print(\"Preprocessed.\")\n  #folds = get_folds(df=df, n_splits=5)\n  #for fold_, (trn_, val_) in enumerate(folds):\n  #    trn_x, trn_y = df.drop(drop_cols, axis=1).iloc[trn_], df['totals.transactionRevenue'].iloc[trn_]\n  #    val_x, val_y = df.drop(drop_cols, axis=1).iloc[val_], df['totals.transactionRevenue'].iloc[val_]\n  if i==0:\n    prev = df\n    print(\"First prev\")\n    i += 1\n    continue\n  trn_x, trn_y = prev.drop(drop_cols, axis=1), prev['totals.transactionRevenue']\n  val_x, val_y = df.drop(drop_cols, axis=1), df['totals.transactionRevenue']\n  print(\"Starting to train...\")\n  lgb_estimator = lgb.train(lgb_params,\n                         init_model=lgb_estimator, # Pass partially trained model\n                         train_set=lgb.Dataset(trn_x, trn_y),\n                         valid_sets=[lgb.Dataset(val_x, val_y)],\n                         valid_names=[\"Valid\"],\n                         early_stopping_rounds = 50,\n                         keep_training_booster=True, # For incremental learning\n                         num_boost_round=70,\n                         verbose_eval=50) # Output after each of 50th time\n  print(\"Continuing...\")\n  prev = df\n  del df, trn_x, trn_y, val_x, val_y\n  gc.collect()\n  i += 1\n\n#trn_x, trn_y = prev.(drop_cols, axis=1), prev['totals.transactionRevenue']\n#lgb_estimator = lgb.train(lgb_params,\n#                         init_model=lgb_estimator, # Pass partially trained model\n#                         train_set=lgb.Dataset(trn_x, trn_y),\n#                         early_stopping_rounds = 50,\n#                         keep_training_booster=True, # For incremental learning\n#                         num_boost_round=70,\n#                         verbose_eval=50) # Output after each of 50th time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b00a1d48f2d754894ea45b8d509bcc0bb3a140f"},"cell_type":"code","source":"del prev\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"y9cC-f2z3U5j","colab_type":"code","colab":{},"trusted":true,"_uuid":"187e4a24c2eb229126c1b39416760d766dbd1b3f"},"cell_type":"code","source":"incremental_test = pd.read_csv(PATH + \"test_v2.csv\",\n                                   dtype={\"fullVisitorId\":\"str\", \"visitId\":\"str\"},\n                                   converters = {col_: json.loads for col_ in dict_columns},\n                                   chunksize=100000)\ntest_df = pd.DataFrame()\nfor df in incremental_test:\n  df = preprocess(df)\n  test_df = pd.concat([test_df, df], axis=0, sort=False).reset_index(drop=True)\n\ndel df\ngc.collect()\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"H1gnks4g1VPB","colab_type":"code","outputId":"cbb2bba9-7e09-46c0-9da3-4565c945341f","colab":{"base_uri":"https://localhost:8080/","height":364},"trusted":true,"_uuid":"a13ac096e4b1822ae02c9d6e996fda63e6404da5"},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error as mse\npreds = lgb_estimator.predict(test_df.drop(drop_cols, axis=1))\n#preds = preprocessing_pipeline['totals.transactionRevenue'][3].inverse_transform(preds.reshape(-1, 1)).reshape(1, -1)[0]\npreds[preds<0] = 0\n\ntrue = test_df['totals.transactionRevenue']\n#true = preprocessing_pipeline['totals.transactionRevenue'][3].inverse_transform(true.values.reshape(-1, 1)).reshape(1, -1)[0]\n\nmse(true, preds) ** 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7dbcdaf3b182d1f5b917d943729cf02f77ed406"},"cell_type":"code","source":"del preds, true, lgb_estimator, test_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"zv11-9iJ5asg","colab_type":"text","_uuid":"b81b9bf9535ab7abef462e269ec9228f915f00e1"},"cell_type":"markdown","source":"# Method 2: Using Dask:\n\n\n\nFor intro on Dask read my post [here](https://towardsdatascience.com/speeding-up-your-algorithms-part-4-dask-7c6ed79994ef).\n\n**Note:**\nYou should only use `Dask` in case of Big Data, where it is not able to fit in your memory."},{"metadata":{"trusted":true,"_uuid":"b43606da77586c773f1553f552173c6dbe129eaf"},"cell_type":"code","source":"%reset -f","execution_count":null,"outputs":[]},{"metadata":{"id":"rctiXDgnZ9tg","colab_type":"code","colab":{},"trusted":true,"_uuid":"58d14c6d75ba49d4865668c73167bd8c312fd3e9"},"cell_type":"code","source":"import dask","execution_count":null,"outputs":[]},{"metadata":{"id":"v0Hcj6ih5aNv","colab_type":"code","outputId":"ed426b21-6dee-4874-a921-b6c14bf5c823","colab":{"base_uri":"https://localhost:8080/","height":123},"trusted":true,"_uuid":"ad218baf9b76c142d7df92c3ddc4125809852013"},"cell_type":"code","source":"import dask.dataframe as dd\nfrom dask.distributed import Client\nclient = Client(processes=False, threads_per_worker=4, n_workers=4, memory_limit='4GB')\nclient","execution_count":null,"outputs":[]},{"metadata":{"id":"rPLfUVQU38ip","colab_type":"code","colab":{},"trusted":true,"_uuid":"48868e819d7ea55433e13d91abbb77cf6760ecee"},"cell_type":"code","source":"dict_cols = [\"device\", \"geoNetwork\", \"totals\", \"trafficSource\"]\ndf = dd.read_csv(PATH+\"train_v2.csv\",\n                dtype={'fullVisitorId': 'str','date': 'str', \n                    **{c: 'str' for c in dict_cols}\n                },\n                parse_dates=['date'], blocksize=1e9)","execution_count":null,"outputs":[]},{"metadata":{"id":"GxrLksfT8_od","colab_type":"code","outputId":"00db78e6-a29f-4a46-d516-ec0c8b23ed8b","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true,"_uuid":"3970a47956583287d94e90224910cdd2af115183"},"cell_type":"code","source":"df.npartitions","execution_count":null,"outputs":[]},{"metadata":{"id":"_p9Uudi--uNO","colab_type":"code","outputId":"139e59c9-b8f7-4f98-efa4-69782e31bc82","colab":{"base_uri":"https://localhost:8080/","height":351},"trusted":true,"_uuid":"2a1ab83b535f7a36962c2da3864ffbf149267f6d"},"cell_type":"code","source":"#df.visualize(size=\"15,15!\")","execution_count":null,"outputs":[]},{"metadata":{"id":"JPX2QoiN--MC","colab_type":"code","outputId":"dd50e404-8f0b-4a15-9798-618b8d62f854","colab":{"base_uri":"https://localhost:8080/","height":195},"trusted":true,"_uuid":"e512a7549ab58baf4f00af494d1381c2c0566a51"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"83MkYBUUOXhc","colab_type":"text","_uuid":"7883970862e442720adc2b8b517ff2ae9a67dc3e"},"cell_type":"markdown","source":"**NOTE:**\n\n`Dask` doesn't have equivalent fucntion of `pandas`'s `json_normalize`. But we can use `Dask`'s `to_bag` function and `bag`'s capability to handle JSON to our advantage."},{"metadata":{"trusted":true,"_uuid":"233b4f71e82ef3783d95aa467226ff1945e3ba7c"},"cell_type":"code","source":"dict_cols = [\"device\", \"geoNetwork\", \"totals\", \"trafficSource\"]\ndrop_columns = [\"hits\", \"socialEngagementType\"]","execution_count":null,"outputs":[]},{"metadata":{"id":"81rpOyD1_cBO","colab_type":"code","colab":{},"trusted":true,"_uuid":"b8f5467a282c5005b6d5ceddb38c6af2c235a98d"},"cell_type":"code","source":"# Convert string Series to dictionary Series\nfor col_ in dict_cols:\n    df[col_] = df[col_].apply(lambda x: eval(x.replace('false', 'False')\n                                                    .replace('true', 'True')\n                                                    .replace('null', 'np.nan')), meta=('', 'object'))","execution_count":null,"outputs":[]},{"metadata":{"id":"aGsdK3ddNjJa","colab_type":"code","colab":{},"trusted":true,"_uuid":"a05468addb6e88bf840354cba72d9a4cf766855f"},"cell_type":"code","source":"final_selected_columns = {\n    \"device\": [\"browser\", \"deviceCategory\", \"isMobile\", \"operatingSystem\"],\n    \"geoNetwork\":[\"city\", \"continent\", \"country\", \"metro\", \"networkDomain\", \"region\", \"subContinent\"],\n    \"totals\":[\"sessionQualityDim\", \"hits\", \"pageviews\", \"timeOnSite\", \"transactionRevenue\"],\n    \"trafficSource\":[\"adContent\", \"adwordsClickInfo.adNetworkType\", \"adwordsClickInfo.page\", \"adwordsClickInfo.slot\",\n                     \"campaign\", \"keyword\",\t\"medium\", \"referralPath\", \"source\"],\n    \"customDimensions\": [\"customDimensions\"],\n    \"channelGrouping\": [\"channelGrouping\"],\n    \"date\":[\"date\"],\n    \"visitNumber\":[\"visitNumber\"],\n    \"visitStartTime\":[\"visitStartTime\"]\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0345a340dd01e2ea635bee640889ddde6e838b77"},"cell_type":"markdown","source":"This method is taken from [here](https://www.kaggle.com/mlisovyi/bigdata-dask-pandas-flat-json-trim-data-upd)."},{"metadata":{"id":"5l6gQLUiOLdd","colab_type":"code","colab":{},"trusted":true,"_uuid":"44b80b1ad6c43064017916b90f55ada5e10b22be"},"cell_type":"code","source":"gc.collect()\nnon_str_cols = ['isMobile', 'isTrueDirect', 'pageviews', 'hits', 'bounces', 'newVisits',\n               'transactionRevenue', 'visits', 'timeOnSite','sessionQualityDim']\ndefault = 0\nfor col_ in dict_cols:\n    for key in final_selected_columns[col_]:\n        if key in non_str_cols: \n            default = 0\n            df[f'{col_}.{key}'] = df[col_].to_bag().pluck(key, default=default).to_dataframe().iloc[:,0]\n            df[f'{col_}.{key}'] = df[f'{col_}.{key}'].astype(int)\n        else: \n            default = \"NaN\"\n            df[f'{col_}.{key}'] = df[col_].to_bag().pluck(key, default=default).to_dataframe().iloc[:,0]\n            df[f'{col_}.{key}'] = df[f'{col_}.{key}'].astype(str)\n    del df[col_]\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"TpDNWPiPPa8X","colab_type":"code","outputId":"9d3caa05-78a6-4ae2-92fa-b202da17a36b","colab":{"base_uri":"https://localhost:8080/","height":195},"trusted":true,"_uuid":"b13d345c908df13bc56787425813e553d907370d"},"cell_type":"code","source":"df = df.drop(drop_columns, axis=1)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"HoXkaG8bBrxs","colab_type":"code","outputId":"97298161-0e51-42fa-bc7c-933a17f2b504","colab":{"base_uri":"https://localhost:8080/","height":539},"trusted":true,"_uuid":"74ea27559a815346c5118bb551ae96606ab83d9b"},"cell_type":"code","source":"#df.visualize(size=\"20,10!\")","execution_count":null,"outputs":[]},{"metadata":{"id":"qmv2aWo1VB98","colab_type":"code","outputId":"53d539bf-2d75-426a-d5d1-b9ce00b59400","colab":{"base_uri":"https://localhost:8080/","height":185},"trusted":true,"_uuid":"51fac729a293756bededa31d383df306a7c6f4a7","scrolled":true},"cell_type":"code","source":"df.isnull().sum().compute()","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"W3vqiTp6m_LV","_uuid":"c780e7eb864b649c409b55e808b8fcea686f97d0"},"cell_type":"markdown","source":"You cannot re-input scaled values from your dataframe directly into its columns (It has to be a series). So, we will Scale Array and directly use it to train our model."},{"metadata":{"colab_type":"code","id":"tJoUglPUm_Kh","outputId":"4d771f3c-2e18-4f4c-c4c2-185a373343fb","colab":{"base_uri":"https://localhost:8080/","height":151},"trusted":true,"_uuid":"c96257fb335024383b8ff305ca35b4eee99b1a15"},"cell_type":"code","source":"# Necessary for converting dataframe to array. Takes specified length from each block.\nlengths = []\nfor part in df.partitions:\n  l = part.shape[0].compute()\n  lengths.append(l)\n  #print(l, part.shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46f0398bedec0e2f38eff72ec66e25a29d59a8a4"},"cell_type":"code","source":"drop_cols =['date', 'visitId', 'fullVisitorId', 'totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36c2985dad3784221863896166beea80c4a6450f"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"hJsEAVBqm_KW","colab":{},"trusted":true,"_uuid":"ce7d249d2cd1b71403517e25bd6c3737a292a6c4"},"cell_type":"code","source":"lcols = [\"customDimensions\", \"channelGrouping\", \"visitStartTime\", \"device.browser\", \"device.deviceCategory\",\n        \"device.operatingSystem\", \"geoNetwork.city\", \"geoNetwork.continent\", \"geoNetwork.metro\", \"geoNetwork.networkDomain\",\n        \"geoNetwork.region\", \"geoNetwork.subContinent\", \"trafficSource.adContent\", \"trafficSource.adwordsClickInfo.adNetworkType\", \n         \"trafficSource.adwordsClickInfo.page\", \"trafficSource.adwordsClickInfo.slot\", \"trafficSource.campaign\",\n        \"trafficSource.keyword\", \"trafficSource.medium\", \"trafficSource.referralPath\", \"trafficSource.source\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fca1f73a843c138674c003dd2f43635d682c66a1"},"cell_type":"code","source":"lcols_index = []\nfor col_ in lcols:\n    lcols_index.append(np.where(col_ == df.columns)[0][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0a5866cdb61017d870f32d15a790df7a350eaf0"},"cell_type":"code","source":"# You cannot assign an array to a column of DataFrame, it has to be a Series.\n# So, we are going to convert our data to arrays, scale and learn from array only.\nX, y = df.drop(drop_cols, axis=1).to_dask_array(lengths=lengths) , df['totals.transactionRevenue'].to_dask_array(lengths=lengths)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc720b7852db177e4718bbd47ff0be8c81672be4"},"cell_type":"code","source":"del df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"ZCfR7qEtm_KD","colab":{},"trusted":true,"_uuid":"204b6b19753af5b0f2516b518519e97637fe0a4a","scrolled":true},"cell_type":"code","source":"#from dask_ml.preprocessing import RobustScaler, LabelEncoder\n#Xo = dask.array.zeros((X.shape[0],1), chunks=(200000,1))\n#for i in range(X.shape[1]):\n#    if i in lcols_index:\n#        lb = LabelEncoder()\n#        temp = lb.fit_transform(X[:i])\n#        Xo = dask.array.concatenate([Xo, temp], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ffbbd9b3e7283545c2c47c703700cd7f14ccab5"},"cell_type":"code","source":"#X = Xo[:, 1:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acd6b79b78e7499abcba57b78df2b7684fb1be19"},"cell_type":"code","source":"#Xo = dask.array.zeros((X.shape[0],1), chunks=(200000,1))\n#for i in range(len(X.shape[1])):\n#  if i == X.shape[1]-1:\n#    rsc = RobustScaler()\n#    y = rsc.fit_transform(y.reshape(-1, 1)).reshape(1, -1)[0]\n#  else:\n#    rsc = RobustScaler()\n#    temp = rsc.fit_transform(X[:,i].reshape(-1, 1))\n#    Xo = dask.array.concatenate([Xo, temp], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca70c3199540e5165d42d46e76b703a64a5acd92"},"cell_type":"code","source":"del X\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"code","id":"j9aL44LLm_Jc","colab":{},"trusted":true,"_uuid":"6482d5717967f15ed317a784553afc6d6fd2474c"},"cell_type":"code","source":"#Xo = Xo[:, 1:]","execution_count":null,"outputs":[]},{"metadata":{"id":"P71Q5JOkTKX3","colab_type":"text","_uuid":"4e653e20d24d3a105d2442b7c3733f43cf5ac9a0"},"cell_type":"markdown","source":"To make blocks for both of equal size. Otherwise you might get broadcast error.\n\n"},{"metadata":{"id":"41fspI44pEZB","colab_type":"code","colab":{},"trusted":true,"_uuid":"eeda455ee0d984b6e989793b7e59b8cc2f0decd9"},"cell_type":"code","source":"#Xo = Xo.rechunk({1: Xo.shape[1]})\n#Xo = Xo.rechunk({0: 200000})\n#y = y.rechunk({0: 200000})","execution_count":null,"outputs":[]},{"metadata":{"id":"7R7k9yXLQXw9","colab_type":"code","outputId":"b4709f0b-7daf-445f-9029-bffdf59552aa","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true,"_uuid":"7d9ad2598659e75d64fef062040eec9dc88655fe"},"cell_type":"code","source":"#tr_len = 0.8*Xo.shape[0]\n#xtrain, ytrain = Xo[:tr_len], y[:tr_len]\n#xvalid, yvalid = Xo[tr_len:], y[tr_len:]\n#xtrain.shape, ytrain.shape, xvalid.shape, yvalid.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"W1khyX7fn3pj","colab_type":"code","colab":{},"trusted":true,"_uuid":"1a9be3f04e98d8fbb78fe6cfab3236ad39512bee"},"cell_type":"code","source":"#from dask_ml.linear_model import LinearRegression","execution_count":null,"outputs":[]},{"metadata":{"id":"-S9OVZV-oKPz","colab_type":"code","colab":{},"trusted":true,"_uuid":"5e4103b2b6aa1224e0c011d15a096bf4a76511ed"},"cell_type":"code","source":"#est = LinearRegression()","execution_count":null,"outputs":[]},{"metadata":{"id":"Vvl4-6Lho-88","colab_type":"code","outputId":"31227917-7150-475f-a011-1513edeb7471","colab":{"base_uri":"https://localhost:8080/","height":84},"trusted":true,"_uuid":"a857a220d1394a53b08c39e11be946d5c4745552"},"cell_type":"code","source":"#est.fit(xtrain, y=ytrain)","execution_count":null,"outputs":[]},{"metadata":{"id":"OnxIlTzdpEqg","colab_type":"code","colab":{},"trusted":true,"_uuid":"20faeb872c94a6b6be2ab4b82cd90e8cee075610"},"cell_type":"code","source":"#preds = est.predict(xvalid)","execution_count":null,"outputs":[]},{"metadata":{"id":"1JR3D7TjqvUP","colab_type":"code","outputId":"091d0aea-1bce-4bad-e569-2312f16878c8","colab":{"base_uri":"https://localhost:8080/","height":50},"trusted":true,"_uuid":"a5bbb3031a6f7fbaac7d3ce3e416996c56d8bb30"},"cell_type":"code","source":"#preds[0:10].compute()","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"HowToHandleBigFilesOnLowMemory-IncrementalLearning.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":["45A-w8c2b3jk","GAEFn6w2cBAy","32g2_Q82nWAi","yJiOYauYePQO"]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}