{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "93b57a34-5e99-cd3c-1477-6e088f042f72"
      },
      "outputs": [],
      "source": [
        "library(data.table)\n",
        "library(digest)\n",
        "dtrain = fread(\"../input/train_date.csv\",drop = \"Id\",   nrows=500000)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "55edcbd1-26f3-48b0-d0a6-6e7201c73f31"
      },
      "source": [
        "Get all the Stations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e373e5bc-4f46-0773-c2f3-02d3bfc9e6cf"
      },
      "outputs": [],
      "source": [
        "statNames = function(y) unique(unlist(lapply(strsplit(grep(\"_\", names(y), value=T), \"_\"), function(x) paste(x[1], x[2], sep=\"_\"))))\n",
        "dstations = statNames(dtrain)\n",
        "print(dstations)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "b279e41b-76e7-b4bd-eb00-3f8e28fd54f3"
      },
      "source": [
        "Find duplicates in each station"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6090c094-8486-e21e-ab7d-6ea7ef38d9f4"
      },
      "outputs": [],
      "source": [
        "yesDuplicated = noDuplicated = c()\n",
        "for(ii in 1:length(dstations)){\n",
        "  selcol = grep(dstations[ii], names(dtrain), value=T) # \n",
        "  dgst = duplicated(lapply(dtrain[,selcol,with=F], digest))\n",
        "  yesDuplicated = c(yesDuplicated, selcol[dgst])\n",
        "  noDuplicated = c(noDuplicated, selcol[!dgst])\n",
        "  gc();gc()\n",
        "}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "12d61f42-12bf-6b30-3bfb-dba30c4c471f"
      },
      "source": [
        "Print them out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ab9e31c3-3f02-ac5e-12b6-eb0182aafde4"
      },
      "outputs": [],
      "source": [
        "noDuplicated"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f7afa17c-01b3-26d8-a524-50219c88cb06"
      },
      "outputs": [],
      "source": [
        "yesDuplicated"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4d2f6a80-cbcb-fd23-8ead-5e04b7d84414"
      },
      "outputs": [],
      "source": [
        "Print them as a vector..."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dd7330d9-0aa7-1426-af94-1e0ddb0270e7"
      },
      "outputs": [],
      "source": [
        "print(paste0(noDuplicated, collapse='\",\"', sep=\"\"))"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "R",
      "language": "R",
      "name": "ir"
    },
    "language_info": {
      "codemirror_mode": "r",
      "file_extension": ".r",
      "mimetype": "text/x-r-source",
      "name": "R",
      "pygments_lexer": "r",
      "version": "3.3.1"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}