{
  "id": 711,
  "title": "Importing to SQL Server and Aggregate Statistics",
  "url": "/competitions/ClaimPredictionChallenge/discussion/711",
  "author_name": "",
  "post_date": "2011-07-14T14:09:06.790Z",
  "votes": null,
  "comment_count": 1,
  "views": 2991,
  "content": "<p>In case it helps others, I'm posting the SQL Server schema I used for this competition. I used this schema in conjunction with SQL Server's import data wizard (i.e. create a new database, then right click on that database and click 'Import Data'). This schema\r\n reflects a fairly compact representation of the data. In addition, I've included many aggregate statistics so that you can verify your import:</p>\r\n<pre>USE ClaimPredictionChallenge;\r\n\r\nCREATE TABLE [test_set](\r\n\t[Row_ID] [int] NOT NULL,\r\n\t[Household_ID] [int] NOT NULL,\r\n\t[Vehicle] [smallint] NOT NULL,\r\n\t[Calendar_Year] [smallint] NOT NULL,\r\n\t[Model_Year] [smallint] NOT NULL,\r\n\t[Blind_Make] [varchar](2) NULL,\r\n\t[Blind_Model] [varchar](6) NULL,\r\n\t[Blind_Submodel] [varchar](8) NULL,\r\n\t[Cat1] [char](1) NOT NULL,\r\n\t[Cat2] [char](1) NOT NULL,\r\n\t[Cat3] [char](1) NOT NULL,\r\n\t[Cat4] [char](1) NOT NULL,\r\n\t[Cat5] [char](1) NOT NULL,\r\n\t[Cat6] [char](1) NOT NULL,\r\n\t[Cat7] [char](1) NOT NULL,\r\n\t[Cat8] [char](1) NOT NULL,\r\n\t[Cat9] [char](1) NOT NULL,\r\n\t[Cat10] [char](1) NOT NULL,\r\n\t[Cat11] [char](1) NOT NULL,\r\n\t[Cat12] [char](1) NOT NULL,\r\n\t[OrdCat] [smallint] NOT NULL,\r\n\t[Var1] [real] NOT NULL,\r\n\t[Var2] [real] NOT NULL,\r\n\t[Var3] [real] NOT NULL,\r\n\t[Var4] [real] NOT NULL,\r\n\t[Var5] [real] NOT NULL,\r\n\t[Var6] [real] NOT NULL,\r\n\t[Var7] [real] NOT NULL,\r\n\t[Var8] [real] NOT NULL,\r\n\t[NVCat] [char](1) NOT NULL,\r\n\t[NVVar1] [real] NOT NULL,\r\n\t[NVVar2] [real] NOT NULL,\r\n\t[NVVar3] [real] NOT NULL,\r\n\t[NVVar4] [real] NOT NULL\r\n);\r\n\r\nCREATE TABLE [train_set](\r\n\t[Row_ID] [int] NOT NULL,\r\n\t[Household_ID] [int] NOT NULL,\r\n\t[Vehicle] [smallint] NOT NULL,\r\n\t[Calendar_Year] [smallint] NOT NULL,\r\n\t[Model_Year] [smallint] NOT NULL,\r\n\t[Blind_Make] [varchar](2) NULL,\r\n\t[Blind_Model] [varchar](6) NULL,\r\n\t[Blind_Submodel] [varchar](8) NULL,\r\n\t[Cat1] [char](1) NULL,\r\n\t[Cat2] [char](1) NULL,\r\n\t[Cat3] [char](1) NULL,\r\n\t[Cat4] [char](1) NULL,\r\n\t[Cat5] [char](1) NULL,\r\n\t[Cat6] [char](1) NULL,\r\n\t[Cat7] [char](1) NULL,\r\n\t[Cat8] [char](1) NULL,\r\n\t[Cat9] [char](1) NULL,\r\n\t[Cat10] [char](1) NULL,\r\n\t[Cat11] [char](1) NULL,\r\n\t[Cat12] [char](1) NULL,\r\n\t[OrdCat] [char](1) NULL,\r\n\t[Var1] [real] NOT NULL,\r\n\t[Var2] [real] NOT NULL,\r\n\t[Var3] [real] NOT NULL,\r\n\t[Var4] [real] NOT NULL,\r\n\t[Var5] [real] NOT NULL,\r\n\t[Var6] [real] NOT NULL,\r\n\t[Var7] [real] NOT NULL,\r\n\t[Var8] [real] NOT NULL,\r\n\t[NVCat] [char](1) NULL,\r\n\t[NVVar1] [real] NOT NULL,\r\n\t[NVVar2] [real] NOT NULL,\r\n\t[NVVar3] [real] NOT NULL,\r\n\t[NVVar4] [real] NOT NULL,\r\n\t[Claim_Amount] [real] NOT NULL\r\n);\r\n</pre>\r\n<pre>-- Make implied NULLs actual NULLs\r\nUPDATE train_set SET Cat1 = NULL WHERE Cat1 = '?';\r\nUPDATE train_set SET Cat2 = NULL WHERE Cat2 = '?';\r\nUPDATE train_set SET Cat3 = NULL WHERE Cat3 = '?';\r\nUPDATE train_set SET Cat4 = NULL WHERE Cat4 = '?';\r\nUPDATE train_set SET Cat5 = NULL WHERE Cat5 = '?';\r\nUPDATE train_set SET Cat6 = NULL WHERE Cat6 = '?';\r\nUPDATE train_set SET Cat7 = NULL WHERE Cat7 = '?';\r\nUPDATE train_set SET Cat8 = NULL WHERE Cat8 = '?';\r\nUPDATE train_set SET Cat9 = NULL WHERE Cat9 = '?';\r\nUPDATE train_set SET Cat10 = NULL WHERE Cat10 = '?';\r\nUPDATE train_set SET Cat11 = NULL WHERE Cat11 = '?';\r\nUPDATE train_set SET Cat12 = NULL WHERE Cat12 IN ('?', '');\r\nUPDATE train_set SET OrdCat = NULL WHERE OrdCat = '?'\r\n\r\n\r\n-- Optionally add primary keys\r\nALTER TABLE train_set ADD CONSTRAINT PK_train_set PRIMARY KEY CLUSTERED (Row_ID);\r\nALTER TABLE test_set ADD CONSTRAINT PK_test_set PRIMARY KEY CLUSTERED (Row_ID);\r\n</pre>\r\n<pre>--------------------------&gt; train_set table</pre>\r\n<pre>SELECT Calendar_Year, COUNT(*) Vehicles_Per_Year FROM train_set GROUP BY Calendar_Year ORDER BY Calendar_Year;\r\n-- Calendar_Year Vehicles_Per_Year\r\n-- 2005\t4025672\r\n-- 2006\t4447730\r\n-- 2007\t4710888\r\n\r\nSELECT Model_Year, COUNT(*) Vehicles_Per_Year FROM train_set GROUP BY Model_Year ORDER BY Model_Year;\r\n--Model_Year\tVehicles_Per_Year\r\n--1981\t20966\r\n--1982\t24868\r\n--1983\t33514\r\n--1984\t53449\r\n--1985\t70738\r\n--1986\t92397\r\n--1987\t104061\r\n--1988\t138717\r\n--1989\t177728\r\n--1990\t203242\r\n--1991\t235688\r\n--1992\t281716\r\n--1993\t382330\r\n--1994\t478067\r\n--1995\t627182\r\n--1996\t589347\r\n--1997\t732651\r\n--1998\t791734\r\n--1999\t887858\r\n--2000\t1004464\r\n--2001\t993400\r\n--2002\t1055299\r\n--2003\t1007184\r\n--2004\t1067981\r\n--2005\t1007447\r\n--2006\t704628\r\n--2007\t366845\r\n--2008\t50787\r\n--2009\t2\r\n</pre>\r\n<pre>SELECT TOP 10 Blind_Make, COUNT(*) Make_Count FROM train_set GROUP BY Blind_Make ORDER BY COUNT(*) DESC\r\n--Blind_Make\tMake_Count\r\n--K\t1657185\r\n--AJ\t1547886\r\n--BW\t1265861\r\n--AU\t1071883\r\n--Y\t848371\r\n--X\t807923\r\n--BO\t657257\r\n--W\t552217\r\n--L\t382047\r\n--AO\t381448\r\n\r\nSELECT TOP 10 Blind_Make, COUNT(*) Make_Count FROM train_set GROUP BY Blind_Make ORDER BY COUNT(*)\r\n--Blind_Make\tMake_Count\r\n--AB\t5\r\n--C\t7\r\n--H\t10\r\n--CA\t13\r\n--A\t17\r\n--AK\t18\r\n--AS\t30\r\n--BK\t40\r\n--BQ\t81\r\n--F\t132\r\n\r\nSELECT TOP 10 Blind_Model, COUNT(*) Model_Count FROM train_set GROUP BY Blind_Model ORDER BY COUNT(*) DESC\r\n--Blind_Model\tModel_Count\r\n--K.7\t597433\r\n--AU.14\t303444\r\n--X.45\t291959\r\n--W.16\t233343\r\n--AU.11\t203473\r\n--BO.38\t194172\r\n--AO.7\t169280\r\n--AJ.58\t159714\r\n--AJ.52\t159204\r\n--AU.58\t151439\r\n\r\nSELECT TOP 10 Blind_Model, COUNT(*) Model_Count FROM train_set GROUP BY Blind_Model ORDER BY COUNT(*), Blind_Model\r\n--Blind_Model\tModel_Count\r\n--AH.114\t1\r\n--AH.117\t1\r\n--AH.13\t1\r\n--AJ.107\t1\r\n--AM.10\t1\r\n--BQ.3\t1\r\n--BQ.9\t1\r\n--BU.37\t1\r\n--BW.123\t1\r\n--BW.128\t1\r\n\r\nSELECT TOP 10 Blind_Submodel, COUNT(*) Vehicle_Count FROM train_set GROUP BY Blind_Submodel ORDER BY COUNT(*) DESC\r\n--Blind_Submodel\tVehicle_Count \r\n--K.7.3\t165298\r\n--AU.58.0\t150110\r\n--AU.14.1\t141627\r\n--AU.14.0\t137043\r\n--W.16.3\t136104\r\n--AU.11.3\t132679\r\n--K.7.0\t116310\r\n--BW.3.0\t112414\r\n--BW.115.0\t112275\r\n--BW.95.0\t106209\r\n\r\nSELECT TOP 10 Blind_Submodel, COUNT(*) AS Vehicle_Count FROM train_set GROUP BY Blind_Submodel ORDER BY COUNT(*), Blind_Submodel\r\n--Blind_Submodel\tVehicle_Count\r\n--AE.3.0\t1\r\n--AE.4.1\t1\r\n--AE.4.2\t1\r\n--AE.6.2\t1\r\n--AH.114.1\t1\r\n--AH.117.1\t1\r\n--AH.13.1\t1\r\n--AJ.107.1\t1\r\n--AM.10.1\t1\r\n--AN.3.1\t1\r\n</pre>\r\n<pre>SELECT Cat1, COUNT(*) AS Total FROM train_set GROUP BY Cat1 ORDER BY COUNT(*) DESC;\r\n--Cat1\tTotal\r\n--B\t4017739\r\n--I\t2654532\r\n--D\t2487951\r\n--F\t1305108\r\n--G\t782602\r\n--A\t768871\r\n--C\t401355\r\n--E\t279699\r\n--J\t233968\r\n--H\t226484\r\n--?\t25981\r\n\r\nSELECT Cat2, COUNT(*) AS Total FROM train_set GROUP BY Cat2 ORDER BY COUNT(*) DESC;\r\n--Cat2\tTotal\r\n--C\t5895027\r\n--?\t4874164\r\n--A\t2191054\r\n--B\t224045\r\n\r\nSELECT Cat3, COUNT(*) AS Total FROM train_set GROUP BY Cat3 ORDER BY COUNT(*) DESC;\r\n--Cat3\tTotal\r\n--A\t7488029\r\n--B\t2256802\r\n--C\t1270889\r\n--E\t886816\r\n--F\t872031\r\n--D\t405724\r\n--?\t3999\r\n\r\nSELECT Cat4, COUNT(*) AS Total FROM train_set GROUP BY Cat4 ORDER BY COUNT(*) DESC;\r\n--Cat4\tTotal\r\n--A\t5723163\r\n--?\t5631649\r\n--C\t1454425\r\n--B\t375053\r\n\r\nSELECT Cat5, COUNT(*) AS Total FROM train_set GROUP BY Cat5 ORDER BY COUNT(*) DESC;\r\n--Cat5\tTotal\r\n--A\t6683980\r\n--?\t5637321\r\n--C\t779280\r\n--B\t83709\r\n\r\nSELECT Cat6, COUNT(*) AS Total FROM train_set GROUP BY Cat6 ORDER BY COUNT(*) DESC;\r\n--Cat6\tTotal\r\n--B\t4265208\r\n--C\t3677694\r\n--D\t3604486\r\n--E\t1173316\r\n--F\t437605\r\n--?\t25981\r\n\r\nSELECT Cat7, COUNT(*) AS Total FROM train_set GROUP BY Cat7 ORDER BY COUNT(*) DESC;\r\n--Cat7\tTotal\r\n--?\t7167634\r\n--C\t4618653\r\n--A\t1050621\r\n--B\t233786\r\n--D\t113596\r\n\r\nSELECT Cat8, COUNT(*) AS Total FROM train_set GROUP BY Cat8 ORDER BY COUNT(*) DESC;\r\n--Cat8\tTotal\r\n--A\t8626513\r\n--B\t3673932\r\n--C\t880481\r\n--?\t3364\r\n\r\nSELECT Cat9, COUNT(*) AS Total FROM train_set GROUP BY Cat9 ORDER BY COUNT(*) DESC;\r\n--Cat9\tTotal\r\n--B\t10850782\r\n--A\t2333508\r\n\r\nSELECT Cat10, COUNT(*) AS Total FROM train_set GROUP BY Cat10 ORDER BY COUNT(*) DESC;\r\n--Cat10\tTotal\r\n--A\t8573092\r\n--B\t3969170\r\n--C\t638111\r\n--?\t3917\r\n\r\nSELECT Cat11, COUNT(*) AS Total FROM train_set GROUP BY Cat11 ORDER BY COUNT(*) DESC;\r\n--Cat11\tTotal\r\n--A\t6951038\r\n--B\t3174528\r\n--C\t1103640\r\n--E\t816595\r\n--F\t787998\r\n--D\t319022\r\n--?\t31469\r\n\r\nSELECT Cat12, COUNT(*) AS Total FROM train_set GROUP BY Cat12 ORDER BY COUNT(*) DESC;\r\n--Cat12\tTotal\r\n--B\t4348276\r\n--C\t3619974\r\n--D\t3525723\r\n--E\t1196458\r\n--F\t462388\r\n--\t28882 (Note: probably should have been '?')\r\n--A\t2589\r\n\r\nSELECT OrdCat, COUNT(*) AS Total FROM train_set GROUP BY OrdCat ORDER BY COUNT(*) DESC;\r\n--OrdCat\tTotal\r\n--4\t5935475\r\n--2\t4146321\r\n--5\t2964704\r\n--3\t93976\r\n--6\t16198\r\n--1\t15835\r\n--?\t7546\r\n--7\t4235\r\n</pre>\r\n<pre>SELECT MIN(Var1) FROM train_set;\r\n-- -2.578222\r\n\r\nSELECT MAX(Var1) FROM train_set;\r\n-- 5.143392\r\n\r\nSELECT SUM(Var1) FROM train_set;\r\n-- -133415.165078922\r\n\r\nSELECT AVG(Var1) FROM train_set;\r\n-- -0.0101192529198707\r\n\r\nSELECT MIN(Var2) FROM train_set;\r\n-- -2.493393\r\n\r\nSELECT MAX(Var2) FROM train_set;\r\n-- 7.82942\r\n\r\nSELECT SUM(Var2) FROM train_set;\r\n-- -858126.221982679\r\n\r\nSELECT AVG(Var2) FROM train_set;\r\n-- -0.0650870256936611\r\n\r\nSELECT MIN(Var3) FROM train_set;\r\n-- -2.790335\r\n\r\nSELECT MAX(Var3) FROM train_set;\r\n-- 5.563325\r\n\r\nSELECT SUM(Var3) FROM train_set;\r\n-- -335328.063572197\r\n\r\nSELECT AVG(Var3) FROM train_set;\r\n-- -0.0254339113878864\r\n\r\nSELECT MIN(Var4) FROM train_set;\r\n-- -2.508216\r\n\r\nSELECT MAX(Var4) FROM train_set;\r\n-- 7.589262\r\n\r\nSELECT SUM(Var4) FROM train_set;\r\n-- -719439.378401036\r\n\r\nSELECT AVG(Var4) FROM train_set;\r\n-- -0.0545679273135707\r\n\r\nSELECT MIN(Var5) FROM train_set;\r\n-- -3.350344\r\n\r\nSELECT MAX(Var5) FROM train_set;\r\n-- 4.018167\r\n\r\nSELECT SUM(Var5) FROM train_set;\r\n-- 50609.1422801865\r\n\r\nSELECT AVG(Var5) FROM train_set;\r\n-- 0.00383859443930515\r\n\r\nSELECT MIN(Var6) FROM train_set;\r\n-- -2.376657\r\n\r\nSELECT MAX(Var6) FROM train_set;\r\n-- 4.584289\r\n\r\nSELECT SUM(Var6) FROM train_set;\r\n-- -528989.514271023\r\n\r\nSELECT AVG(Var6) FROM train_set;\r\n-- -0.0401227153127717\r\n\r\nSELECT MIN(Var7) FROM train_set;\r\n-- -2.778491\r\n\r\nSELECT MAX(Var7) FROM train_set;\r\n-- 4.127148\r\nSELECT SUM(Var7) FROM train_set;\r\n-- -319229.602963614\r\n\r\nSELECT AVG(Var7) FROM train_set;\r\n-- -0.024212877823805\r\n\r\nSELECT MIN(Var8) FROM train_set;\r\n-- -2.163042\r\n\r\nSELECT MAX(Var8) FROM train_set;\r\n-- 47.35074\r\nSELECT SUM(Var8) FROM train_set;\r\n-- -772079.797664134\r\n\r\nSELECT AVG(Var8) FROM train_set;\r\n-- -0.0585605897370381\r\n</pre>\r\n<pre>SELECT NVCat, COUNT(*) AS NVCat_Count FROM train_set GROUP BY NVCat ORDER BY COUNT(*) DESC;\r\n\r\n--NVCat\tNVCat_Count\r\n--M\t5767944\r\n--O\t3416948\r\n--N\t1328428\r\n--L\t804000\r\n--J\t559165\r\n--E\t401274\r\n--F\t325556\r\n--B\t173724\r\n--H\t134702\r\n--K\t119996\r\n--C\t64753\r\n--A\t45758\r\n--I\t19208\r\n--G\t16073\r\n--D\t6761\r\n</pre>\r\n<pre>SELECT MIN(NVVar1) FROM train_set;\r\n-- -0.2315299\r\n\r\nSELECT MAX(NVVar1) FROM train_set;\r\n-- 6.62711\r\n\r\nSELECT SUM(NVVar1) FROM train_set;\r\n-- 193599.357427523\r\n\r\nSELECT AVG(NVVar1) FROM train_set;\r\n-- 0.0146840942839943\r\n\r\nSELECT MIN(NVVar2) FROM train_set;\r\n-- -0.2661168\r\n\r\nSELECT MAX(NVVar2) FROM train_set;\r\n-- 8.883081\r\n\r\nSELECT SUM(NVVar2) FROM train_set;\r\n-- 230879.161265343\r\n\r\nSELECT AVG(NVVar2) FROM train_set;\r\n-- 0.0175116871113532\r\n\r\nSELECT MIN(NVVar3) FROM train_set;\r\n-- -0.2723372\r\n\r\nSELECT MAX(NVVar3) FROM train_set;\r\n-- 8.691144\r\n\r\nSELECT SUM(NVVar3) FROM train_set;\r\n-- 178545.106917024\r\n\r\nSELECT AVG(NVVar3) FROM train_set;\r\n-- 0.0135422618068188\r\n\r\nSELECT MIN(NVVar4) FROM train_set;\r\n-- -0.2514189\r\n\r\nSELECT MAX(NVVar4) FROM train_set;\r\n-- 6.388803\r\n\r\nSELECT SUM(NVVar4) FROM train_set;\r\n-- 244090.89812161\r\n\r\nSELECT AVG(NVVar4) FROM train_set;\r\n-- 0.0185137688962857\r\n\r\n</pre>\r\n<pre>SELECT MIN(Claim_Amount) FROM train_set;\r\n-- 0\r\n\r\nSELECT MAX(Claim_Amount) FROM train_set;\r\n-- 11440.75\r\n\r\nSELECT SUM(Claim_Amount) FROM train_set;\r\n-- 17939315.1737089\r\n\r\nSELECT AVG(Claim_Amount) FROM train_set;\r\n-- 1.36065841798905\r\n\r\nSELECT CAST((SELECT COUNT(*) FROM train_set WHERE Claim_Amount = 0) AS real) / (SELECT COUNT(*) FROM train_set);\r\n-- 0.9927486\r\n</pre>\r\n<pre>--------------------------&gt; test_set table</pre>\r\n<pre>SELECT Calendar_Year, COUNT(*) Vehicles_Per_Year FROM test_set GROUP BY Calendar_Year ORDER BY Calendar_Year;\r\n--Calendar_Year\tVehicles_Per_Year\r\n--2008\t2118739\r\n--2009\t2196126\r\n\r\nSELECT Model_Year, COUNT(*) Vehicles_Per_Year FROM test_set GROUP BY Model_Year ORDER BY Model_Year;\r\n--Model_Year\tVehicles_Per_Year\r\n--1984\t1\r\n--1986\t1\r\n--1987\t3\r\n--1989\t261\r\n--1990\t2\r\n--1993\t2\r\n--1994\t3110\r\n--1995\t9870\r\n--1996\t4486\r\n--1997\t1910\r\n--1998\t49561\r\n--1999\t300904\r\n--2000\t403069\r\n--2001\t374678\r\n--2002\t405638\r\n--2003\t395663\r\n--2004\t387160\r\n--2005\t459951\r\n--2006\t488643\r\n--2007\t471198\r\n--2008\t378999\r\n--2009\t153706\r\n--2010\t26049\r\n</pre>\r\n<pre>SELECT TOP 100 Blind_Make, COUNT(*) Make_Count FROM test_set GROUP BY Blind_Make ORDER BY COUNT(*) DESC\r\n--Blind_Make\tMake_Count\r\n--K\t554407\r\n--AU\t413009\r\n--Y\t362378\r\n--AO\t308624\r\n--X\t300597\r\n--BF\t161520\r\n--L\t161133\r\n--BO\t138123\r\n--BU\t135777\r\n--AL\t118783\r\n\r\nSELECT TOP 10 Blind_Make, COUNT(*) Make_Count FROM test_set GROUP BY Blind_Make ORDER BY COUNT(*)\r\n--Blind_Make\tMake_Count\r\n--AS\t31\r\n--CB\t36\r\n--Z\t42\r\n--BQ\t70\r\n--AG\t105\r\n--T\t286\r\n--E\t328\r\n--BA\t335\r\n--AX\t853\r\n--BR\t2796\r\n\r\nSELECT TOP 10 Blind_Model, COUNT(*) Model_Count FROM test_set GROUP BY Blind_Model ORDER BY COUNT(*) DESC\r\n--Blind_Model\tModel_Count\r\n--K.7\t328245\r\n--AO.2\t277386\r\n--AU.14\t205493\r\n--X.45\t129039\r\n--Y.29\t125310\r\n--K.65\t101414\r\n--BO.38\t94234\r\n--Y.34\t90861\r\n--X.24\t88772\r\n--AU.11\t86631\r\n\r\nSELECT TOP 10 Blind_Model, COUNT(*) Model_Count FROM test_set GROUP BY Blind_Model ORDER BY COUNT(*), Blind_Model\r\n--Blind_Model\tModel_Count\r\n--AL.5\t1\r\n--BH.17\t1\r\n--BZ.18\t1\r\n--BZ.19\t1\r\n--AG.1\t4\r\n--E.18\t4\r\n--AY.68\t5\r\n--AG.5\t6\r\n--BG.24\t7\r\n--BT.68\t7\r\n\r\nSELECT TOP 10 Blind_Submodel, COUNT(*) Vehicle_Count FROM test_set GROUP BY Blind_Submodel ORDER BY COUNT(*) DESC\r\n--Blind_Submodel\tVehicle_Count\r\n--AU.14.1\t148261\r\n--K.7.3\t139655\r\n--AU.58.0\t77481\r\n--AO.2.5\t67926\r\n--AO.2.13\t66457\r\n--K.7.1\t63079\r\n--X.45.8\t57204\r\n--K.7.2\t56568\r\n--AO.2.11\t55175\r\n--Y.29.0\t54375\r\n\r\nSELECT TOP 10 Blind_Submodel, COUNT(*) AS Vehicle_Count FROM test_set GROUP BY Blind_Submodel ORDER BY COUNT(*), Blind_Submodel\r\n--Blind_Submodel\tVehicle_Count\r\n--AL.5.0\t1\r\n--AR.2.1\t1\r\n--AY.21.2\t1\r\n--AY.32.1\t1\r\n--AY.33.3\t1\r\n--AY.57.1\t1\r\n--AY.60.3\t1\r\n--AZ.27.18\t1\r\n--BH.17.0\t1\r\n--BZ.18.1\t1\r\n</pre>\r\n<pre>SELECT Cat1, COUNT(*) AS Total FROM test_set GROUP BY Cat1 ORDER BY COUNT(*) DESC;\r\n--Cat1\tTotal\r\n--B\t2858631\r\n--A\t572074\r\n--G\t457935\r\n--E\t283406\r\n--C\t75064\r\n--F\t67755\r\n\r\nSELECT Cat2, COUNT(*) AS Total FROM test_set GROUP BY Cat2 ORDER BY COUNT(*) DESC;\r\n--Cat2\tTotal\r\n--C\t3302271\r\n--A\t891344\r\n--B\t121250\r\n\r\nSELECT Cat3, COUNT(*) AS Total FROM test_set GROUP BY Cat3 ORDER BY COUNT(*) DESC;\r\n--Cat3\tTotal\r\n--B\t2156278\r\n--A\t2007717\r\n--F\t148523\r\n--D\t1998\r\n--E\t260\r\n--C\t89\r\n\r\nSELECT Cat4, COUNT(*) AS Total FROM test_set GROUP BY Cat4 ORDER BY COUNT(*) DESC;\r\n--A\t4145055\r\n--C\t169810\r\n\r\nSELECT Cat5, COUNT(*) AS Total FROM test_set GROUP BY Cat5 ORDER BY COUNT(*) DESC;\r\n--Cat5\tTotal\r\n--A\t3804513\r\n--C\t458019\r\n--B\t52333\r\n\r\nSELECT Cat6, COUNT(*) AS Total FROM test_set GROUP BY Cat6 ORDER BY COUNT(*) DESC;\r\n--Cat6\tTotal\r\n--B\t1934060\r\n--D\t999375\r\n--C\t890884\r\n--E\t384563\r\n--F\t105697\r\n--A\t286\r\n\r\nSELECT Cat7, COUNT(*) AS Total FROM test_set GROUP BY Cat7 ORDER BY COUNT(*) DESC;\r\n--Cat7\tTotal\r\n--C\t3378427\r\n--A\t761065\r\n--B\t162285\r\n--D\t13088\r\n\r\nSELECT Cat8, COUNT(*) AS Total FROM test_set GROUP BY Cat8 ORDER BY COUNT(*) DESC;\r\n--Cat8\tTotal\r\n--A\t2238788\r\n--B\t1824053\r\n--C\t252024\r\n\r\nSELECT Cat9, COUNT(*) AS Total FROM test_set GROUP BY Cat9 ORDER BY COUNT(*) DESC;\r\n--Cat9\tTotal\r\n--B\t2508005\r\n--A\t1806860\r\n\r\nSELECT Cat10, COUNT(*) AS Total FROM test_set GROUP BY Cat10 ORDER BY COUNT(*) DESC;\r\n--Cat10\tTotal\r\n--A\t2807036\r\n--B\t1299078\r\n--C\t208751\r\n\r\nSELECT Cat11, COUNT(*) AS Total FROM test_set GROUP BY Cat11 ORDER BY COUNT(*) DESC;\r\n--Cat11\tTotal\r\n--A\t2279692\r\n--B\t1041526\r\n--C\t361464\r\n--E\t268073\r\n--F\t258831\r\n--D\t105279\r\n\r\nSELECT Cat12, COUNT(*) AS Total FROM test_set GROUP BY Cat12 ORDER BY COUNT(*) DESC;\r\n--Cat12\tTotal\r\n--B\t1426443\r\n--C\t1185201\r\n--D\t1157767\r\n--E\t393609\r\n--F\t150976\r\n--A\t869\r\n\r\nSELECT OrdCat, COUNT(*) AS Total FROM test_set GROUP BY OrdCat ORDER BY COUNT(*) DESC;\r\n--OrdCat\tTotal\r\n--2\t2033886\r\n--4\t1950006\r\n--5\t258913\r\n--3\t69039\r\n--7\t1619\r\n--6\t1115\r\n--1\t287\r\n</pre>\r\n<pre>SELECT MIN(Var1) FROM test_set;\r\n-- -3.09246\r\n\r\nSELECT MAX(Var1) FROM test_set;\r\n-- 3.086437\r\n\r\nSELECT SUM(Var1) FROM test_set;\r\n-- -1256558.88539529\r\n\r\nSELECT AVG(Var1) FROM test_set;\r\n-- -0.291216268735011\r\n\r\nSELECT MIN(Var2) FROM test_set;\r\n-- -2.14757\r\n\r\nSELECT MAX(Var2) FROM test_set;\r\n-- 7.82942\r\n\r\nSELECT SUM(Var2) FROM test_set;\r\n-- -228227.168241829\r\n\r\nSELECT AVG(Var2) FROM test_set;\r\n-- -0.0528932349544723\r\n\r\nSELECT MIN(Var3) FROM test_set;\r\n-- -2.46637\r\n\r\nSELECT MAX(Var3) FROM test_set;\r\n-- 2.069135\r\n\r\nSELECT SUM(Var3) FROM test_set;\r\n-- -900919.478849606\r\n\r\nSELECT AVG(Var3) FROM test_set;\r\n-- -0.20879436062301\r\n\r\nSELECT MIN(Var4) FROM test_set;\r\n-- -2.169942\r\n\r\nSELECT MAX(Var4) FROM test_set;\r\n-- 7.94445\r\n\r\nSELECT SUM(Var4) FROM test_set;\r\n-- -383728.20701796\r\n\r\nSELECT AVG(Var4) FROM test_set;\r\n-- -0.0889316831506803\r\n\r\nSELECT MIN(Var5) FROM test_set;\r\n-- -5.057174\r\n\r\nSELECT MAX(Var5) FROM test_set;\r\n-- 2.876315\r\n\r\nSELECT SUM(Var5) FROM test_set;\r\n-- -745186.335228794\r\n\r\nSELECT AVG(Var5) FROM test_set;\r\n-- -0.17270212051334\r\n\r\nSELECT MIN(Var6) FROM test_set;\r\n-- -2.029253\r\n\r\nSELECT MAX(Var6) FROM test_set;\r\n-- 2.858966\r\n\r\nSELECT SUM(Var6) FROM test_set;\r\n-- -1602296.64387677\r\n\r\nSELECT AVG(Var6) FROM test_set;\r\n-- -0.371343400981669\r\n\r\nSELECT MIN(Var7) FROM test_set;\r\n-- -2.21326\r\n\r\nSELECT MAX(Var7) FROM test_set;\r\n-- 1.681913\r\n\r\nSELECT SUM(Var7) FROM test_set;\r\n-- -2385324.70575756\r\n\r\nSELECT AVG(Var7) FROM test_set;\r\n-- -0.552815605067032\r\n\r\nSELECT MIN(Var8) FROM test_set;\r\n-- -1.484801\r\n\r\nSELECT MAX(Var8) FROM test_set;\r\n-- 46.72172\r\n\r\nSELECT SUM(Var8) FROM test_set;\r\n-- 378444.673914099\r\n\r\nSELECT AVG(Var8) FROM test_set;\r\n-- 0.0877071875746051\r\n</pre>\r\n<pre>SELECT NVCat, COUNT(*) AS NVCat_Count FROM test_set GROUP BY NVCat ORDER BY COUNT(*) DESC;\r\n\r\n--NVCat\tNVCat_Count\r\n--M\t1861275\r\n--O\t1205736\r\n--N\t407912\r\n--L\t284602\r\n--J\t178823\r\n--E\t120624\r\n--F\t85545\r\n--B\t57172\r\n--H\t41202\r\n--K\t33773\r\n--C\t16057\r\n--A\t12100\r\n--G\t4408\r\n--I\t4159\r\n--D\t1477\r\n</pre>\r\n<pre>SELECT MIN(NVVar1) FROM test_set;\r\n-- -0.2315299\r\n\r\nSELECT MAX(NVVar1) FROM test_set;\r\n-- 6.62711\r\n\r\nSELECT SUM(NVVar1) FROM test_set;\r\n-- -116860.566231743\r\n\r\nSELECT AVG(NVVar1) FROM test_set;\r\n-- -0.0270832497034652\r\n\r\nSELECT MIN(NVVar2) FROM test_set;\r\n-- -0.2661168\r\n\r\nSELECT MAX(NVVar2) FROM test_set;\r\n-- 8.883081\r\n\r\nSELECT SUM(NVVar2) FROM test_set;\r\n-- -43259.1929190159\r\n\r\nSELECT AVG(NVVar2) FROM test_set;\r\n-- -0.0100256190909834\r\n\r\nSELECT MIN(NVVar3) FROM test_set;\r\n-- -0.2723372\r\n\r\nSELECT MAX(NVVar3) FROM test_set;\r\n-- 8.691144\r\n\r\nSELECT SUM(NVVar3) FROM test_set;\r\n-- -195504.770742655\r\n\r\nSELECT AVG(NVVar3) FROM test_set;\r\n-- -0.0453095915498294\r\n\r\nSELECT MIN(NVVar4) FROM test_set;\r\n-- -0.2514189\r\n\r\nSELECT MAX(NVVar4) FROM test_set;\r\n-- 6.388803\r\n\r\nSELECT SUM(NVVar4) FROM test_set;\r\n-- 41278.680568099\r\n\r\nSELECT AVG(NVVar4) FROM test_set;\r\n-- 0.00956662156709399\r\n</pre>",
  "messages": [
    {
      "id": "4605",
      "postDate": "07/14/2011 14:09:06",
      "content": "<p>In case it helps others, I'm posting the SQL Server schema I used for this competition. I used this schema in conjunction with SQL Server's import data wizard (i.e. create a new database, then right click on that database and click 'Import Data'). This schema\r\n reflects a fairly compact representation of the data. In addition, I've included many aggregate statistics so that you can verify your import:</p>\r\n<pre>USE ClaimPredictionChallenge;\r\n\r\nCREATE TABLE [test_set](\r\n\t[Row_ID] [int] NOT NULL,\r\n\t[Household_ID] [int] NOT NULL,\r\n\t[Vehicle] [smallint] NOT NULL,\r\n\t[Calendar_Year] [smallint] NOT NULL,\r\n\t[Model_Year] [smallint] NOT NULL,\r\n\t[Blind_Make] [varchar](2) NULL,\r\n\t[Blind_Model] [varchar](6) NULL,\r\n\t[Blind_Submodel] [varchar](8) NULL,\r\n\t[Cat1] [char](1) NOT NULL,\r\n\t[Cat2] [char](1) NOT NULL,\r\n\t[Cat3] [char](1) NOT NULL,\r\n\t[Cat4] [char](1) NOT NULL,\r\n\t[Cat5] [char](1) NOT NULL,\r\n\t[Cat6] [char](1) NOT NULL,\r\n\t[Cat7] [char](1) NOT NULL,\r\n\t[Cat8] [char](1) NOT NULL,\r\n\t[Cat9] [char](1) NOT NULL,\r\n\t[Cat10] [char](1) NOT NULL,\r\n\t[Cat11] [char](1) NOT NULL,\r\n\t[Cat12] [char](1) NOT NULL,\r\n\t[OrdCat] [smallint] NOT NULL,\r\n\t[Var1] [real] NOT NULL,\r\n\t[Var2] [real] NOT NULL,\r\n\t[Var3] [real] NOT NULL,\r\n\t[Var4] [real] NOT NULL,\r\n\t[Var5] [real] NOT NULL,\r\n\t[Var6] [real] NOT NULL,\r\n\t[Var7] [real] NOT NULL,\r\n\t[Var8] [real] NOT NULL,\r\n\t[NVCat] [char](1) NOT NULL,\r\n\t[NVVar1] [real] NOT NULL,\r\n\t[NVVar2] [real] NOT NULL,\r\n\t[NVVar3] [real] NOT NULL,\r\n\t[NVVar4] [real] NOT NULL\r\n);\r\n\r\nCREATE TABLE [train_set](\r\n\t[Row_ID] [int] NOT NULL,\r\n\t[Household_ID] [int] NOT NULL,\r\n\t[Vehicle] [smallint] NOT NULL,\r\n\t[Calendar_Year] [smallint] NOT NULL,\r\n\t[Model_Year] [smallint] NOT NULL,\r\n\t[Blind_Make] [varchar](2) NULL,\r\n\t[Blind_Model] [varchar](6) NULL,\r\n\t[Blind_Submodel] [varchar](8) NULL,\r\n\t[Cat1] [char](1) NULL,\r\n\t[Cat2] [char](1) NULL,\r\n\t[Cat3] [char](1) NULL,\r\n\t[Cat4] [char](1) NULL,\r\n\t[Cat5] [char](1) NULL,\r\n\t[Cat6] [char](1) NULL,\r\n\t[Cat7] [char](1) NULL,\r\n\t[Cat8] [char](1) NULL,\r\n\t[Cat9] [char](1) NULL,\r\n\t[Cat10] [char](1) NULL,\r\n\t[Cat11] [char](1) NULL,\r\n\t[Cat12] [char](1) NULL,\r\n\t[OrdCat] [char](1) NULL,\r\n\t[Var1] [real] NOT NULL,\r\n\t[Var2] [real] NOT NULL,\r\n\t[Var3] [real] NOT NULL,\r\n\t[Var4] [real] NOT NULL,\r\n\t[Var5] [real] NOT NULL,\r\n\t[Var6] [real] NOT NULL,\r\n\t[Var7] [real] NOT NULL,\r\n\t[Var8] [real] NOT NULL,\r\n\t[NVCat] [char](1) NULL,\r\n\t[NVVar1] [real] NOT NULL,\r\n\t[NVVar2] [real] NOT NULL,\r\n\t[NVVar3] [real] NOT NULL,\r\n\t[NVVar4] [real] NOT NULL,\r\n\t[Claim_Amount] [real] NOT NULL\r\n);\r\n</pre>\r\n<pre>-- Make implied NULLs actual NULLs\r\nUPDATE train_set SET Cat1 = NULL WHERE Cat1 = '?';\r\nUPDATE train_set SET Cat2 = NULL WHERE Cat2 = '?';\r\nUPDATE train_set SET Cat3 = NULL WHERE Cat3 = '?';\r\nUPDATE train_set SET Cat4 = NULL WHERE Cat4 = '?';\r\nUPDATE train_set SET Cat5 = NULL WHERE Cat5 = '?';\r\nUPDATE train_set SET Cat6 = NULL WHERE Cat6 = '?';\r\nUPDATE train_set SET Cat7 = NULL WHERE Cat7 = '?';\r\nUPDATE train_set SET Cat8 = NULL WHERE Cat8 = '?';\r\nUPDATE train_set SET Cat9 = NULL WHERE Cat9 = '?';\r\nUPDATE train_set SET Cat10 = NULL WHERE Cat10 = '?';\r\nUPDATE train_set SET Cat11 = NULL WHERE Cat11 = '?';\r\nUPDATE train_set SET Cat12 = NULL WHERE Cat12 IN ('?', '');\r\nUPDATE train_set SET OrdCat = NULL WHERE OrdCat = '?'\r\n\r\n\r\n-- Optionally add primary keys\r\nALTER TABLE train_set ADD CONSTRAINT PK_train_set PRIMARY KEY CLUSTERED (Row_ID);\r\nALTER TABLE test_set ADD CONSTRAINT PK_test_set PRIMARY KEY CLUSTERED (Row_ID);\r\n</pre>\r\n<pre>--------------------------&gt; train_set table</pre>\r\n<pre>SELECT Calendar_Year, COUNT(*) Vehicles_Per_Year FROM train_set GROUP BY Calendar_Year ORDER BY Calendar_Year;\r\n-- Calendar_Year Vehicles_Per_Year\r\n-- 2005\t4025672\r\n-- 2006\t4447730\r\n-- 2007\t4710888\r\n\r\nSELECT Model_Year, COUNT(*) Vehicles_Per_Year FROM train_set GROUP BY Model_Year ORDER BY Model_Year;\r\n--Model_Year\tVehicles_Per_Year\r\n--1981\t20966\r\n--1982\t24868\r\n--1983\t33514\r\n--1984\t53449\r\n--1985\t70738\r\n--1986\t92397\r\n--1987\t104061\r\n--1988\t138717\r\n--1989\t177728\r\n--1990\t203242\r\n--1991\t235688\r\n--1992\t281716\r\n--1993\t382330\r\n--1994\t478067\r\n--1995\t627182\r\n--1996\t589347\r\n--1997\t732651\r\n--1998\t791734\r\n--1999\t887858\r\n--2000\t1004464\r\n--2001\t993400\r\n--2002\t1055299\r\n--2003\t1007184\r\n--2004\t1067981\r\n--2005\t1007447\r\n--2006\t704628\r\n--2007\t366845\r\n--2008\t50787\r\n--2009\t2\r\n</pre>\r\n<pre>SELECT TOP 10 Blind_Make, COUNT(*) Make_Count FROM train_set GROUP BY Blind_Make ORDER BY COUNT(*) DESC\r\n--Blind_Make\tMake_Count\r\n--K\t1657185\r\n--AJ\t1547886\r\n--BW\t1265861\r\n--AU\t1071883\r\n--Y\t848371\r\n--X\t807923\r\n--BO\t657257\r\n--W\t552217\r\n--L\t382047\r\n--AO\t381448\r\n\r\nSELECT TOP 10 Blind_Make, COUNT(*) Make_Count FROM train_set GROUP BY Blind_Make ORDER BY COUNT(*)\r\n--Blind_Make\tMake_Count\r\n--AB\t5\r\n--C\t7\r\n--H\t10\r\n--CA\t13\r\n--A\t17\r\n--AK\t18\r\n--AS\t30\r\n--BK\t40\r\n--BQ\t81\r\n--F\t132\r\n\r\nSELECT TOP 10 Blind_Model, COUNT(*) Model_Count FROM train_set GROUP BY Blind_Model ORDER BY COUNT(*) DESC\r\n--Blind_Model\tModel_Count\r\n--K.7\t597433\r\n--AU.14\t303444\r\n--X.45\t291959\r\n--W.16\t233343\r\n--AU.11\t203473\r\n--BO.38\t194172\r\n--AO.7\t169280\r\n--AJ.58\t159714\r\n--AJ.52\t159204\r\n--AU.58\t151439\r\n\r\nSELECT TOP 10 Blind_Model, COUNT(*) Model_Count FROM train_set GROUP BY Blind_Model ORDER BY COUNT(*), Blind_Model\r\n--Blind_Model\tModel_Count\r\n--AH.114\t1\r\n--AH.117\t1\r\n--AH.13\t1\r\n--AJ.107\t1\r\n--AM.10\t1\r\n--BQ.3\t1\r\n--BQ.9\t1\r\n--BU.37\t1\r\n--BW.123\t1\r\n--BW.128\t1\r\n\r\nSELECT TOP 10 Blind_Submodel, COUNT(*) Vehicle_Count FROM train_set GROUP BY Blind_Submodel ORDER BY COUNT(*) DESC\r\n--Blind_Submodel\tVehicle_Count \r\n--K.7.3\t165298\r\n--AU.58.0\t150110\r\n--AU.14.1\t141627\r\n--AU.14.0\t137043\r\n--W.16.3\t136104\r\n--AU.11.3\t132679\r\n--K.7.0\t116310\r\n--BW.3.0\t112414\r\n--BW.115.0\t112275\r\n--BW.95.0\t106209\r\n\r\nSELECT TOP 10 Blind_Submodel, COUNT(*) AS Vehicle_Count FROM train_set GROUP BY Blind_Submodel ORDER BY COUNT(*), Blind_Submodel\r\n--Blind_Submodel\tVehicle_Count\r\n--AE.3.0\t1\r\n--AE.4.1\t1\r\n--AE.4.2\t1\r\n--AE.6.2\t1\r\n--AH.114.1\t1\r\n--AH.117.1\t1\r\n--AH.13.1\t1\r\n--AJ.107.1\t1\r\n--AM.10.1\t1\r\n--AN.3.1\t1\r\n</pre>\r\n<pre>SELECT Cat1, COUNT(*) AS Total FROM train_set GROUP BY Cat1 ORDER BY COUNT(*) DESC;\r\n--Cat1\tTotal\r\n--B\t4017739\r\n--I\t2654532\r\n--D\t2487951\r\n--F\t1305108\r\n--G\t782602\r\n--A\t768871\r\n--C\t401355\r\n--E\t279699\r\n--J\t233968\r\n--H\t226484\r\n--?\t25981\r\n\r\nSELECT Cat2, COUNT(*) AS Total FROM train_set GROUP BY Cat2 ORDER BY COUNT(*) DESC;\r\n--Cat2\tTotal\r\n--C\t5895027\r\n--?\t4874164\r\n--A\t2191054\r\n--B\t224045\r\n\r\nSELECT Cat3, COUNT(*) AS Total FROM train_set GROUP BY Cat3 ORDER BY COUNT(*) DESC;\r\n--Cat3\tTotal\r\n--A\t7488029\r\n--B\t2256802\r\n--C\t1270889\r\n--E\t886816\r\n--F\t872031\r\n--D\t405724\r\n--?\t3999\r\n\r\nSELECT Cat4, COUNT(*) AS Total FROM train_set GROUP BY Cat4 ORDER BY COUNT(*) DESC;\r\n--Cat4\tTotal\r\n--A\t5723163\r\n--?\t5631649\r\n--C\t1454425\r\n--B\t375053\r\n\r\nSELECT Cat5, COUNT(*) AS Total FROM train_set GROUP BY Cat5 ORDER BY COUNT(*) DESC;\r\n--Cat5\tTotal\r\n--A\t6683980\r\n--?\t5637321\r\n--C\t779280\r\n--B\t83709\r\n\r\nSELECT Cat6, COUNT(*) AS Total FROM train_set GROUP BY Cat6 ORDER BY COUNT(*) DESC;\r\n--Cat6\tTotal\r\n--B\t4265208\r\n--C\t3677694\r\n--D\t3604486\r\n--E\t1173316\r\n--F\t437605\r\n--?\t25981\r\n\r\nSELECT Cat7, COUNT(*) AS Total FROM train_set GROUP BY Cat7 ORDER BY COUNT(*) DESC;\r\n--Cat7\tTotal\r\n--?\t7167634\r\n--C\t4618653\r\n--A\t1050621\r\n--B\t233786\r\n--D\t113596\r\n\r\nSELECT Cat8, COUNT(*) AS Total FROM train_set GROUP BY Cat8 ORDER BY COUNT(*) DESC;\r\n--Cat8\tTotal\r\n--A\t8626513\r\n--B\t3673932\r\n--C\t880481\r\n--?\t3364\r\n\r\nSELECT Cat9, COUNT(*) AS Total FROM train_set GROUP BY Cat9 ORDER BY COUNT(*) DESC;\r\n--Cat9\tTotal\r\n--B\t10850782\r\n--A\t2333508\r\n\r\nSELECT Cat10, COUNT(*) AS Total FROM train_set GROUP BY Cat10 ORDER BY COUNT(*) DESC;\r\n--Cat10\tTotal\r\n--A\t8573092\r\n--B\t3969170\r\n--C\t638111\r\n--?\t3917\r\n\r\nSELECT Cat11, COUNT(*) AS Total FROM train_set GROUP BY Cat11 ORDER BY COUNT(*) DESC;\r\n--Cat11\tTotal\r\n--A\t6951038\r\n--B\t3174528\r\n--C\t1103640\r\n--E\t816595\r\n--F\t787998\r\n--D\t319022\r\n--?\t31469\r\n\r\nSELECT Cat12, COUNT(*) AS Total FROM train_set GROUP BY Cat12 ORDER BY COUNT(*) DESC;\r\n--Cat12\tTotal\r\n--B\t4348276\r\n--C\t3619974\r\n--D\t3525723\r\n--E\t1196458\r\n--F\t462388\r\n--\t28882 (Note: probably should have been '?')\r\n--A\t2589\r\n\r\nSELECT OrdCat, COUNT(*) AS Total FROM train_set GROUP BY OrdCat ORDER BY COUNT(*) DESC;\r\n--OrdCat\tTotal\r\n--4\t5935475\r\n--2\t4146321\r\n--5\t2964704\r\n--3\t93976\r\n--6\t16198\r\n--1\t15835\r\n--?\t7546\r\n--7\t4235\r\n</pre>\r\n<pre>SELECT MIN(Var1) FROM train_set;\r\n-- -2.578222\r\n\r\nSELECT MAX(Var1) FROM train_set;\r\n-- 5.143392\r\n\r\nSELECT SUM(Var1) FROM train_set;\r\n-- -133415.165078922\r\n\r\nSELECT AVG(Var1) FROM train_set;\r\n-- -0.0101192529198707\r\n\r\nSELECT MIN(Var2) FROM train_set;\r\n-- -2.493393\r\n\r\nSELECT MAX(Var2) FROM train_set;\r\n-- 7.82942\r\n\r\nSELECT SUM(Var2) FROM train_set;\r\n-- -858126.221982679\r\n\r\nSELECT AVG(Var2) FROM train_set;\r\n-- -0.0650870256936611\r\n\r\nSELECT MIN(Var3) FROM train_set;\r\n-- -2.790335\r\n\r\nSELECT MAX(Var3) FROM train_set;\r\n-- 5.563325\r\n\r\nSELECT SUM(Var3) FROM train_set;\r\n-- -335328.063572197\r\n\r\nSELECT AVG(Var3) FROM train_set;\r\n-- -0.0254339113878864\r\n\r\nSELECT MIN(Var4) FROM train_set;\r\n-- -2.508216\r\n\r\nSELECT MAX(Var4) FROM train_set;\r\n-- 7.589262\r\n\r\nSELECT SUM(Var4) FROM train_set;\r\n-- -719439.378401036\r\n\r\nSELECT AVG(Var4) FROM train_set;\r\n-- -0.0545679273135707\r\n\r\nSELECT MIN(Var5) FROM train_set;\r\n-- -3.350344\r\n\r\nSELECT MAX(Var5) FROM train_set;\r\n-- 4.018167\r\n\r\nSELECT SUM(Var5) FROM train_set;\r\n-- 50609.1422801865\r\n\r\nSELECT AVG(Var5) FROM train_set;\r\n-- 0.00383859443930515\r\n\r\nSELECT MIN(Var6) FROM train_set;\r\n-- -2.376657\r\n\r\nSELECT MAX(Var6) FROM train_set;\r\n-- 4.584289\r\n\r\nSELECT SUM(Var6) FROM train_set;\r\n-- -528989.514271023\r\n\r\nSELECT AVG(Var6) FROM train_set;\r\n-- -0.0401227153127717\r\n\r\nSELECT MIN(Var7) FROM train_set;\r\n-- -2.778491\r\n\r\nSELECT MAX(Var7) FROM train_set;\r\n-- 4.127148\r\nSELECT SUM(Var7) FROM train_set;\r\n-- -319229.602963614\r\n\r\nSELECT AVG(Var7) FROM train_set;\r\n-- -0.024212877823805\r\n\r\nSELECT MIN(Var8) FROM train_set;\r\n-- -2.163042\r\n\r\nSELECT MAX(Var8) FROM train_set;\r\n-- 47.35074\r\nSELECT SUM(Var8) FROM train_set;\r\n-- -772079.797664134\r\n\r\nSELECT AVG(Var8) FROM train_set;\r\n-- -0.0585605897370381\r\n</pre>\r\n<pre>SELECT NVCat, COUNT(*) AS NVCat_Count FROM train_set GROUP BY NVCat ORDER BY COUNT(*) DESC;\r\n\r\n--NVCat\tNVCat_Count\r\n--M\t5767944\r\n--O\t3416948\r\n--N\t1328428\r\n--L\t804000\r\n--J\t559165\r\n--E\t401274\r\n--F\t325556\r\n--B\t173724\r\n--H\t134702\r\n--K\t119996\r\n--C\t64753\r\n--A\t45758\r\n--I\t19208\r\n--G\t16073\r\n--D\t6761\r\n</pre>\r\n<pre>SELECT MIN(NVVar1) FROM train_set;\r\n-- -0.2315299\r\n\r\nSELECT MAX(NVVar1) FROM train_set;\r\n-- 6.62711\r\n\r\nSELECT SUM(NVVar1) FROM train_set;\r\n-- 193599.357427523\r\n\r\nSELECT AVG(NVVar1) FROM train_set;\r\n-- 0.0146840942839943\r\n\r\nSELECT MIN(NVVar2) FROM train_set;\r\n-- -0.2661168\r\n\r\nSELECT MAX(NVVar2) FROM train_set;\r\n-- 8.883081\r\n\r\nSELECT SUM(NVVar2) FROM train_set;\r\n-- 230879.161265343\r\n\r\nSELECT AVG(NVVar2) FROM train_set;\r\n-- 0.0175116871113532\r\n\r\nSELECT MIN(NVVar3) FROM train_set;\r\n-- -0.2723372\r\n\r\nSELECT MAX(NVVar3) FROM train_set;\r\n-- 8.691144\r\n\r\nSELECT SUM(NVVar3) FROM train_set;\r\n-- 178545.106917024\r\n\r\nSELECT AVG(NVVar3) FROM train_set;\r\n-- 0.0135422618068188\r\n\r\nSELECT MIN(NVVar4) FROM train_set;\r\n-- -0.2514189\r\n\r\nSELECT MAX(NVVar4) FROM train_set;\r\n-- 6.388803\r\n\r\nSELECT SUM(NVVar4) FROM train_set;\r\n-- 244090.89812161\r\n\r\nSELECT AVG(NVVar4) FROM train_set;\r\n-- 0.0185137688962857\r\n\r\n</pre>\r\n<pre>SELECT MIN(Claim_Amount) FROM train_set;\r\n-- 0\r\n\r\nSELECT MAX(Claim_Amount) FROM train_set;\r\n-- 11440.75\r\n\r\nSELECT SUM(Claim_Amount) FROM train_set;\r\n-- 17939315.1737089\r\n\r\nSELECT AVG(Claim_Amount) FROM train_set;\r\n-- 1.36065841798905\r\n\r\nSELECT CAST((SELECT COUNT(*) FROM train_set WHERE Claim_Amount = 0) AS real) / (SELECT COUNT(*) FROM train_set);\r\n-- 0.9927486\r\n</pre>\r\n<pre>--------------------------&gt; test_set table</pre>\r\n<pre>SELECT Calendar_Year, COUNT(*) Vehicles_Per_Year FROM test_set GROUP BY Calendar_Year ORDER BY Calendar_Year;\r\n--Calendar_Year\tVehicles_Per_Year\r\n--2008\t2118739\r\n--2009\t2196126\r\n\r\nSELECT Model_Year, COUNT(*) Vehicles_Per_Year FROM test_set GROUP BY Model_Year ORDER BY Model_Year;\r\n--Model_Year\tVehicles_Per_Year\r\n--1984\t1\r\n--1986\t1\r\n--1987\t3\r\n--1989\t261\r\n--1990\t2\r\n--1993\t2\r\n--1994\t3110\r\n--1995\t9870\r\n--1996\t4486\r\n--1997\t1910\r\n--1998\t49561\r\n--1999\t300904\r\n--2000\t403069\r\n--2001\t374678\r\n--2002\t405638\r\n--2003\t395663\r\n--2004\t387160\r\n--2005\t459951\r\n--2006\t488643\r\n--2007\t471198\r\n--2008\t378999\r\n--2009\t153706\r\n--2010\t26049\r\n</pre>\r\n<pre>SELECT TOP 100 Blind_Make, COUNT(*) Make_Count FROM test_set GROUP BY Blind_Make ORDER BY COUNT(*) DESC\r\n--Blind_Make\tMake_Count\r\n--K\t554407\r\n--AU\t413009\r\n--Y\t362378\r\n--AO\t308624\r\n--X\t300597\r\n--BF\t161520\r\n--L\t161133\r\n--BO\t138123\r\n--BU\t135777\r\n--AL\t118783\r\n\r\nSELECT TOP 10 Blind_Make, COUNT(*) Make_Count FROM test_set GROUP BY Blind_Make ORDER BY COUNT(*)\r\n--Blind_Make\tMake_Count\r\n--AS\t31\r\n--CB\t36\r\n--Z\t42\r\n--BQ\t70\r\n--AG\t105\r\n--T\t286\r\n--E\t328\r\n--BA\t335\r\n--AX\t853\r\n--BR\t2796\r\n\r\nSELECT TOP 10 Blind_Model, COUNT(*) Model_Count FROM test_set GROUP BY Blind_Model ORDER BY COUNT(*) DESC\r\n--Blind_Model\tModel_Count\r\n--K.7\t328245\r\n--AO.2\t277386\r\n--AU.14\t205493\r\n--X.45\t129039\r\n--Y.29\t125310\r\n--K.65\t101414\r\n--BO.38\t94234\r\n--Y.34\t90861\r\n--X.24\t88772\r\n--AU.11\t86631\r\n\r\nSELECT TOP 10 Blind_Model, COUNT(*) Model_Count FROM test_set GROUP BY Blind_Model ORDER BY COUNT(*), Blind_Model\r\n--Blind_Model\tModel_Count\r\n--AL.5\t1\r\n--BH.17\t1\r\n--BZ.18\t1\r\n--BZ.19\t1\r\n--AG.1\t4\r\n--E.18\t4\r\n--AY.68\t5\r\n--AG.5\t6\r\n--BG.24\t7\r\n--BT.68\t7\r\n\r\nSELECT TOP 10 Blind_Submodel, COUNT(*) Vehicle_Count FROM test_set GROUP BY Blind_Submodel ORDER BY COUNT(*) DESC\r\n--Blind_Submodel\tVehicle_Count\r\n--AU.14.1\t148261\r\n--K.7.3\t139655\r\n--AU.58.0\t77481\r\n--AO.2.5\t67926\r\n--AO.2.13\t66457\r\n--K.7.1\t63079\r\n--X.45.8\t57204\r\n--K.7.2\t56568\r\n--AO.2.11\t55175\r\n--Y.29.0\t54375\r\n\r\nSELECT TOP 10 Blind_Submodel, COUNT(*) AS Vehicle_Count FROM test_set GROUP BY Blind_Submodel ORDER BY COUNT(*), Blind_Submodel\r\n--Blind_Submodel\tVehicle_Count\r\n--AL.5.0\t1\r\n--AR.2.1\t1\r\n--AY.21.2\t1\r\n--AY.32.1\t1\r\n--AY.33.3\t1\r\n--AY.57.1\t1\r\n--AY.60.3\t1\r\n--AZ.27.18\t1\r\n--BH.17.0\t1\r\n--BZ.18.1\t1\r\n</pre>\r\n<pre>SELECT Cat1, COUNT(*) AS Total FROM test_set GROUP BY Cat1 ORDER BY COUNT(*) DESC;\r\n--Cat1\tTotal\r\n--B\t2858631\r\n--A\t572074\r\n--G\t457935\r\n--E\t283406\r\n--C\t75064\r\n--F\t67755\r\n\r\nSELECT Cat2, COUNT(*) AS Total FROM test_set GROUP BY Cat2 ORDER BY COUNT(*) DESC;\r\n--Cat2\tTotal\r\n--C\t3302271\r\n--A\t891344\r\n--B\t121250\r\n\r\nSELECT Cat3, COUNT(*) AS Total FROM test_set GROUP BY Cat3 ORDER BY COUNT(*) DESC;\r\n--Cat3\tTotal\r\n--B\t2156278\r\n--A\t2007717\r\n--F\t148523\r\n--D\t1998\r\n--E\t260\r\n--C\t89\r\n\r\nSELECT Cat4, COUNT(*) AS Total FROM test_set GROUP BY Cat4 ORDER BY COUNT(*) DESC;\r\n--A\t4145055\r\n--C\t169810\r\n\r\nSELECT Cat5, COUNT(*) AS Total FROM test_set GROUP BY Cat5 ORDER BY COUNT(*) DESC;\r\n--Cat5\tTotal\r\n--A\t3804513\r\n--C\t458019\r\n--B\t52333\r\n\r\nSELECT Cat6, COUNT(*) AS Total FROM test_set GROUP BY Cat6 ORDER BY COUNT(*) DESC;\r\n--Cat6\tTotal\r\n--B\t1934060\r\n--D\t999375\r\n--C\t890884\r\n--E\t384563\r\n--F\t105697\r\n--A\t286\r\n\r\nSELECT Cat7, COUNT(*) AS Total FROM test_set GROUP BY Cat7 ORDER BY COUNT(*) DESC;\r\n--Cat7\tTotal\r\n--C\t3378427\r\n--A\t761065\r\n--B\t162285\r\n--D\t13088\r\n\r\nSELECT Cat8, COUNT(*) AS Total FROM test_set GROUP BY Cat8 ORDER BY COUNT(*) DESC;\r\n--Cat8\tTotal\r\n--A\t2238788\r\n--B\t1824053\r\n--C\t252024\r\n\r\nSELECT Cat9, COUNT(*) AS Total FROM test_set GROUP BY Cat9 ORDER BY COUNT(*) DESC;\r\n--Cat9\tTotal\r\n--B\t2508005\r\n--A\t1806860\r\n\r\nSELECT Cat10, COUNT(*) AS Total FROM test_set GROUP BY Cat10 ORDER BY COUNT(*) DESC;\r\n--Cat10\tTotal\r\n--A\t2807036\r\n--B\t1299078\r\n--C\t208751\r\n\r\nSELECT Cat11, COUNT(*) AS Total FROM test_set GROUP BY Cat11 ORDER BY COUNT(*) DESC;\r\n--Cat11\tTotal\r\n--A\t2279692\r\n--B\t1041526\r\n--C\t361464\r\n--E\t268073\r\n--F\t258831\r\n--D\t105279\r\n\r\nSELECT Cat12, COUNT(*) AS Total FROM test_set GROUP BY Cat12 ORDER BY COUNT(*) DESC;\r\n--Cat12\tTotal\r\n--B\t1426443\r\n--C\t1185201\r\n--D\t1157767\r\n--E\t393609\r\n--F\t150976\r\n--A\t869\r\n\r\nSELECT OrdCat, COUNT(*) AS Total FROM test_set GROUP BY OrdCat ORDER BY COUNT(*) DESC;\r\n--OrdCat\tTotal\r\n--2\t2033886\r\n--4\t1950006\r\n--5\t258913\r\n--3\t69039\r\n--7\t1619\r\n--6\t1115\r\n--1\t287\r\n</pre>\r\n<pre>SELECT MIN(Var1) FROM test_set;\r\n-- -3.09246\r\n\r\nSELECT MAX(Var1) FROM test_set;\r\n-- 3.086437\r\n\r\nSELECT SUM(Var1) FROM test_set;\r\n-- -1256558.88539529\r\n\r\nSELECT AVG(Var1) FROM test_set;\r\n-- -0.291216268735011\r\n\r\nSELECT MIN(Var2) FROM test_set;\r\n-- -2.14757\r\n\r\nSELECT MAX(Var2) FROM test_set;\r\n-- 7.82942\r\n\r\nSELECT SUM(Var2) FROM test_set;\r\n-- -228227.168241829\r\n\r\nSELECT AVG(Var2) FROM test_set;\r\n-- -0.0528932349544723\r\n\r\nSELECT MIN(Var3) FROM test_set;\r\n-- -2.46637\r\n\r\nSELECT MAX(Var3) FROM test_set;\r\n-- 2.069135\r\n\r\nSELECT SUM(Var3) FROM test_set;\r\n-- -900919.478849606\r\n\r\nSELECT AVG(Var3) FROM test_set;\r\n-- -0.20879436062301\r\n\r\nSELECT MIN(Var4) FROM test_set;\r\n-- -2.169942\r\n\r\nSELECT MAX(Var4) FROM test_set;\r\n-- 7.94445\r\n\r\nSELECT SUM(Var4) FROM test_set;\r\n-- -383728.20701796\r\n\r\nSELECT AVG(Var4) FROM test_set;\r\n-- -0.0889316831506803\r\n\r\nSELECT MIN(Var5) FROM test_set;\r\n-- -5.057174\r\n\r\nSELECT MAX(Var5) FROM test_set;\r\n-- 2.876315\r\n\r\nSELECT SUM(Var5) FROM test_set;\r\n-- -745186.335228794\r\n\r\nSELECT AVG(Var5) FROM test_set;\r\n-- -0.17270212051334\r\n\r\nSELECT MIN(Var6) FROM test_set;\r\n-- -2.029253\r\n\r\nSELECT MAX(Var6) FROM test_set;\r\n-- 2.858966\r\n\r\nSELECT SUM(Var6) FROM test_set;\r\n-- -1602296.64387677\r\n\r\nSELECT AVG(Var6) FROM test_set;\r\n-- -0.371343400981669\r\n\r\nSELECT MIN(Var7) FROM test_set;\r\n-- -2.21326\r\n\r\nSELECT MAX(Var7) FROM test_set;\r\n-- 1.681913\r\n\r\nSELECT SUM(Var7) FROM test_set;\r\n-- -2385324.70575756\r\n\r\nSELECT AVG(Var7) FROM test_set;\r\n-- -0.552815605067032\r\n\r\nSELECT MIN(Var8) FROM test_set;\r\n-- -1.484801\r\n\r\nSELECT MAX(Var8) FROM test_set;\r\n-- 46.72172\r\n\r\nSELECT SUM(Var8) FROM test_set;\r\n-- 378444.673914099\r\n\r\nSELECT AVG(Var8) FROM test_set;\r\n-- 0.0877071875746051\r\n</pre>\r\n<pre>SELECT NVCat, COUNT(*) AS NVCat_Count FROM test_set GROUP BY NVCat ORDER BY COUNT(*) DESC;\r\n\r\n--NVCat\tNVCat_Count\r\n--M\t1861275\r\n--O\t1205736\r\n--N\t407912\r\n--L\t284602\r\n--J\t178823\r\n--E\t120624\r\n--F\t85545\r\n--B\t57172\r\n--H\t41202\r\n--K\t33773\r\n--C\t16057\r\n--A\t12100\r\n--G\t4408\r\n--I\t4159\r\n--D\t1477\r\n</pre>\r\n<pre>SELECT MIN(NVVar1) FROM test_set;\r\n-- -0.2315299\r\n\r\nSELECT MAX(NVVar1) FROM test_set;\r\n-- 6.62711\r\n\r\nSELECT SUM(NVVar1) FROM test_set;\r\n-- -116860.566231743\r\n\r\nSELECT AVG(NVVar1) FROM test_set;\r\n-- -0.0270832497034652\r\n\r\nSELECT MIN(NVVar2) FROM test_set;\r\n-- -0.2661168\r\n\r\nSELECT MAX(NVVar2) FROM test_set;\r\n-- 8.883081\r\n\r\nSELECT SUM(NVVar2) FROM test_set;\r\n-- -43259.1929190159\r\n\r\nSELECT AVG(NVVar2) FROM test_set;\r\n-- -0.0100256190909834\r\n\r\nSELECT MIN(NVVar3) FROM test_set;\r\n-- -0.2723372\r\n\r\nSELECT MAX(NVVar3) FROM test_set;\r\n-- 8.691144\r\n\r\nSELECT SUM(NVVar3) FROM test_set;\r\n-- -195504.770742655\r\n\r\nSELECT AVG(NVVar3) FROM test_set;\r\n-- -0.0453095915498294\r\n\r\nSELECT MIN(NVVar4) FROM test_set;\r\n-- -0.2514189\r\n\r\nSELECT MAX(NVVar4) FROM test_set;\r\n-- 6.388803\r\n\r\nSELECT SUM(NVVar4) FROM test_set;\r\n-- 41278.680568099\r\n\r\nSELECT AVG(NVVar4) FROM test_set;\r\n-- 0.00956662156709399\r\n</pre>",
      "rawMarkdown": "",
      "votes": null
    },
    {
      "id": "4619",
      "postDate": "07/14/2011 22:53:59",
      "content": "<p>Hey Jeff,</p>\r\n<p>I have written a tool that will generate all this SQL code for you, and also the sql to load the data so you don't have to use the import wizard. This makes it a lot more transferable as everything is in the script and you don't have to rely on manual wizards.</p>\r\n<p>You can get the tool - it is contained in my package Tiberius, but is a stand alone exe that works on its own with no licence requirements.</p>\r\n<p>Examples on how to use it are in my blog</p>\r\n<p><a href=\"http://anotherdataminingblog.blogspot.com/2011/05/progress-loading-hhp-data.html\">http://anotherdataminingblog.blogspot.com/2011/05/progress-loading-hhp-data.html</a></p>\r\n<p>you can download it from <a href=\"http://www.tiberius.biz\">www.tiberius.biz</a></p>\r\n<p>&nbsp;</p>",
      "rawMarkdown": "",
      "votes": null
    }
  ],
  "comments": [
    {
      "id": 4619,
      "author_name": "salimali",
      "author_url": "",
      "post_date": "07/14/2011 22:53:59",
      "content": "<p>Hey Jeff,</p>\r\n<p>I have written a tool that will generate all this SQL code for you, and also the sql to load the data so you don't have to use the import wizard. This makes it a lot more transferable as everything is in the script and you don't have to rely on manual wizards.</p>\r\n<p>You can get the tool - it is contained in my package Tiberius, but is a stand alone exe that works on its own with no licence requirements.</p>\r\n<p>Examples on how to use it are in my blog</p>\r\n<p><a href=\"http://anotherdataminingblog.blogspot.com/2011/05/progress-loading-hhp-data.html\">http://anotherdataminingblog.blogspot.com/2011/05/progress-loading-hhp-data.html</a></p>\r\n<p>you can download it from <a href=\"http://www.tiberius.biz\">www.tiberius.biz</a></p>\r\n<p>&nbsp;</p>",
      "votes": null,
      "replies": []
    }
  ],
  "raw_markdown_by_id": {
    "4605": "",
    "4619": ""
  },
  "source": "meta"
}