{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "a3a51f3d-c103-5b5b-8052-96d228a316b3"
      },
      "source": [
        "A compact way of summarizing the date dataset are row-wise summaries of total time, number of steps, step times."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8533497b-cf2d-cbbf-b302-19626be16c75"
      },
      "outputs": [],
      "source": [
        "#My first try here\n",
        "\n",
        "require(data.table)\n",
        "require(dplyr)\n",
        "require(pbapply) #nicer apply with status bar\n",
        "\n",
        "dtDate = fread(\"../input/train_date.csv\",nrows = 100000) #full date data takes ~30min to summarize \n",
        "\n",
        "dateId = dtDate$Id\n",
        "dtDate = dtDate[, Id := NULL]\n",
        "\n",
        "#Summary function defined here - these are exemplary features\n",
        "getSummaryOfRow = function(v)\n",
        "{\n",
        "    v[!(is.na(v))] %>% length() -> numberOfSteps\n",
        "    v[!(is.na(v))] %>% sort() %>% diff() -> v_diffed\n",
        "    v_diffed[v_diffed>0] %>% sum() -> totalTime\n",
        "    v_diffed[v_diffed>0] %>% length() -> numberOfRealSteps\n",
        "    if(numberOfRealSteps>0) #some Processes take no time at all\n",
        "        {\n",
        "            v_diffed[v_diffed>0] %>% max() -> longestStep\n",
        "            v_diffed[v_diffed>0] %>% min() -> shortestStep       \n",
        "        }\n",
        "    else\n",
        "        {\n",
        "            longestStep = -1\n",
        "            shortestStep = -1\n",
        "        }\n",
        "    return (c(numberOfSteps,totalTime,longestStep,shortestStep,numberOfRealSteps))\n",
        "}\n",
        "\n",
        " #row-wise application of summary function, subsequent transpose to retain data structure\n",
        "dateSummary = as.data.table(t(pbapply(dtDate, 1, getSummaryOfRow)))\n",
        "colnames(dateSummary) = c(\"numberOfSteps\",\"totalTime\",\"longestStep\",\"shortestStep\",\"numberOfRealSteps\")\n",
        "dateSummary$Id = dateId\n",
        "\n",
        "head(dateSummary)\n",
        "\n",
        "#can be merged to other features via Id"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "9d38278a-2501-bc83-d2b4-4c6c89d20825"
      },
      "source": [
        "Possible extensions include normalization, product family comparisons, other aggregates..."
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "R",
      "language": "R",
      "name": "ir"
    },
    "language_info": {
      "codemirror_mode": "r",
      "file_extension": ".r",
      "mimetype": "text/x-r-source",
      "name": "R",
      "pygments_lexer": "r",
      "version": "3.3.1"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}