{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def trainmerge(trainZ,mergeB,onvar):\n    return trainZ.merge(pd.read_csv('../input/ashrae-energy-prediction/'+mergeB+'.csv'), on=onvar, how='left')\n\ndef loaddata(trainM,xi,splitten):\n    if splitten:\n        trainX=trainmerge(trainmerge(pd.read_csv('../input/ashrae-energy-prediction/'+trainM+'.csv')[xi*11000000:(xi+1)*11000000],'building_metadata','building_id'),'weather_'+trainM,['site_id', 'timestamp'])\n    else:\n        trainX=trainmerge(trainmerge(pd.read_csv('../input/ashrae-energy-prediction/'+trainM+'.csv'),'building_metadata','building_id'),'weather_'+trainM,['site_id', 'timestamp'])\n    \n    print(trainX.info())\n    numericcolumns=['air_temperature','cloud_coverage','dew_temperature','floor_count','meter','precip_depth_1_hr','sea_level_pressure','wind_direction','wind_speed','year_built']\n\n    return trainX\n\n\ntrain=loaddata('train',0,False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EnergypUse=train[['primary_use','meter_reading']].groupby('primary_use').mean().sort_values('meter_reading')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#todo split date in month\n\ndef HotEnc(totaal,codes,indexv,label,trainvw):\n    #add variables\n    totaal['M2K']=totaal['square_feet'].values*(25-totaal['air_temperature'].values)\n    totaal['pM2K']=1/totaal['M2K']\n    if trainvw:\n        totaal['EpM2']=totaal['meter_reading'].fillna(0)/totaal['square_feet'].values  #energy use is function of surface...\n        #totaal['EpM2K']=totaal['meter_reading'].fillna(0)*totaal['pM2K']\n        #totaal['EpM2K']=totaal['EpM2K'].fillna(0).replace(np.inf,0)\n        #totaal['EpM2K']=totaal['EpM2K'].astype('int')\n    totaal['floor_count'] = totaal['floor_count'].fillna(0).astype(np.int8)\n    totaal['year_built'] = totaal['year_built'].fillna(0).astype(np.int16)\n\n    totaal['age'] = (2019-totaal['year_built'].fillna(totaal['year_built'].median())).astype(np.int8)\n    totaal[\"timestamp\"] = pd.to_datetime(totaal[\"timestamp\"], format='%Y-%m-%d %H:%M:%S')\n    totaal['month_datetime'] = totaal['timestamp'].dt.month.astype(np.int8)\n    totaal['weekofyear_datetime'] = totaal['timestamp'].dt.weekofyear.astype(np.int8)\n    totaal['dayofyear_datetime'] = totaal['timestamp'].dt.dayofyear.astype(np.int16)\n    totaal['hour_datetime'] = totaal['timestamp'].dt.hour.astype(np.int8)  \n    totaal['day_week'] = totaal['timestamp'].dt.dayofweek.astype(np.int8)\n    totaal['day_month_datetime'] = totaal['timestamp'].dt.day.astype(np.int8)\n    totaal['year_built'] = totaal['year_built']-1900\n    totaal['square_feet'] = np.log(totaal['square_feet'])\n     \n    # One Hot Encode target mean()\n    codesdict=codes[label].to_dict()\n    totaal['primary_use']=totaal['primary_use'].map(codesdict)\n    \n    print(totaal.info())\n    return totaal.drop(['timestamp','precip_depth_1_hr'],axis=1)\n\ntrain=HotEnc(train,EnergypUse,'leeg','meter_reading',True)\n#test=loaddata('test')\n#test=HotEnc(test,train,'leeg','meter_reading',False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clustertechniques2(dtrain,label,indexv):\n    print('#encodings',dtrain.shape)\n    cols=[ci for ci in dtrain.columns if ci not in [indexv,'index',label]]\n    dtest=dtrain[dtrain[label].isnull()==True][[indexv,label]]\n    print(dtest)\n\n    print('encodings  after shape',dtrain.shape)\n    #split data or use splitted data\n    X_train=dtrain[dtrain[label].isnull()==False].drop([indexv,label],axis=1).fillna(0)\n    Y_train=dtrain[dtrain[label].isnull()==False][label]\n    X_test=dtrain[dtrain[label].isnull()==True].drop([indexv,label],axis=1).fillna(0)\n    Y_test=np.random.random((X_test.shape[0],1))\n    if len(X_test)==0:\n        from sklearn.model_selection import train_test_split\n        X_train,X_test,Y_train,Y_test = train_test_split(dtrain.drop(label,axis=1).fillna(0),dtrain[label],test_size=0.25,random_state=0)\n    lenxtr=len(X_train)\n    print('splitting data train test X-y',X_train.shape,Y_train.shape,X_test.shape,Y_test.shape)\n   \n\n\n    import matplotlib.pyplot as plt \n    from sklearn import preprocessing\n    scale = preprocessing.MinMaxScaler().fit(X_train)\n    X_train = scale.transform(X_train)\n    X_test = scale.transform(X_test)\n\n        \n    from sklearn.decomposition import PCA,TruncatedSVD,NMF,FastICA\n    from umap import UMAP  # knn lookalike of tSNE but faster, so scales up\n    from sklearn.manifold import TSNE #limit number of records to 100000\n\n    clusters = [Dummy(1),\n                PCA(n_components=0.7,random_state=0,whiten=True),\n                #FastICA(n_components=7,random_state=0),\n                TruncatedSVD(n_components=10, n_iter=7, random_state=42),\n                #NMF(n_components=10,random_state=0),            \n                #UMAP(n_neighbors=5,n_components=10, min_dist=0.3,metric='minkowski'),\n                #TSNE(n_components=2,random_state=0)\n                ] \n    clunaam=[\"raw\",'PCA','tSVD','UMAP']#,'ICA','tSVD','nmf','UMAP','tSNE']\n    \n    \n    from sklearn.neighbors import KNeighborsClassifier,KNeighborsRegressor\n    from sklearn.svm import SVC, LinearSVC,NuSVC\n    from sklearn.multiclass import OneVsRestClassifier\n    from sklearn.ensemble import RandomForestRegressor,ExtraTreesRegressor, RandomForestClassifier,ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier\n    from sklearn.neural_network import MLPClassifier,MLPRegressor\n    from sklearn.linear_model import PassiveAggressiveClassifier,Perceptron,SGDClassifier,LogisticRegression\n    import xgboost as xgb\n    from sklearn.tree import DecisionTreeClassifier\n    from sklearn.naive_bayes import GaussianNB\n    from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis\n    from sklearn.linear_model import ElasticNetCV,ridge_regression,HuberRegressor,LinearRegression,BayesianRidge,RANSACRegressor\n    from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor\n    from sklearn.metrics import mean_squared_error,r2_score\n    classifiers = [\n\n                   #GradientBoostingRegressor(),\n                   ExtraTreesRegressor(),\n                   RandomForestRegressor(random_state=1, n_estimators=10),        \n                   BayesianRidge(),\n        \n                   RANSACRegressor(),\n                   KNeighborsRegressor(),\n                   ElasticNetCV(cv=5, random_state=0),\n                   HuberRegressor(),\n                   LinearRegression(),\n                  ]\n    clanaam= ['xTreer','rFor','BaysR','Ransac','KNNr','elast','huber','linear',]\n    from sklearn.metrics import classification_report,confusion_matrix,accuracy_score\n    \n    results=[]\n\n\n    #cluster data\n    for clu in clusters:\n        clunm=clunaam[clusters.index(clu)] #find naam\n        X_total_clu = clu.fit_transform(np.concatenate( (X_train,X_test),axis=0))\n        X_total_clu=np.concatenate((X_total_clu,np.concatenate( (X_train,X_test),axis=0)),axis=1)\n        print(X_total_clu.shape)\n        plt.scatter(X_total_clu[:lenxtr,0],X_total_clu[:lenxtr,1],c=Y_train.values,cmap='prism')\n        plt.title(clu)\n        plt.show()\n        \n        #classifiy \n        for cla in classifiers:\n            import datetime\n            start = datetime.datetime.now()\n            clanm=clanaam[classifiers.index(cla)] #find naam\n            \n            print('    ',cla)\n            cla.fit(X_total_clu,np.concatenate( (Y_train,Y_test)) )\n            cla.fit(X_total_clu[:lenxtr],Y_train )\n            \n            #predict\n            trainpredi=cla.predict(X_total_clu[:lenxtr])\n\n            #print(classification_report(trainpredi,Y_train))            \n            testpredi=cla.predict(X_total_clu[lenxtr:])  \n            if classifiers.index(cla) in [0,2,3,4,5,7,8,9,10,11,12,13]:\n                trainprediprob=cla.predict(X_total_clu[:lenxtr])\n                testprediprob=cla.predict(X_total_clu[lenxtr:]) \n                \n                plt.scatter(x=testprediprob, y=testpredi, marker='.', alpha=0.53)\n                plt.show()            \n            #testpredi=converging(pd.DataFrame(X_train),pd.DataFrame(X_test),Y_train,pd.DataFrame(testpredi),Y_test,clu,cla) #PCA(n_components=10,random_state=0,whiten=True),MLPClassifier(alpha=0.510,activation='logistic'))\n            \n            if len(dtest)==0:\n                test_score=cla.score(X_total_clu[lenxtr:],Y_test)\n                mse = mean_squared_error(testpredi,Y_test)\n                train_score=cla.score(X_total_clu[:lenxtr],Y_train)\n\n                li = [clunm,clanm,train_score,test_score,mse]\n                results.append(li)\n                r2s=r2_score(testpredi,Y_test)  \n                print(r2s)\n\n                plt.title(clanm+'test mse versus unknown:'+np.str(test_score)+' '+np.str(mse)+' and test confusionmatrix')\n                plt.scatter(x=Y_test, y=testpredi, marker='.', alpha=1)\n                plt.scatter(x=[np.mean(Y_test)], y=[np.mean(testpredi)], marker='o', color='red')\n                plt.xlabel('Real test'); plt.ylabel('Pred. test')\n                plt.show()\n\n\n            else:\n#                testpredlabel=le.inverse_transform(testpredi)  #use if you labellezid the classes \n                testpredlabel=testpredi\n                print(confusion_matrix(trainpredi,Y_train))\n                submit = pd.DataFrame({indexv: dtest[indexv],label: testpredlabel})\n                submit[label]=submit[label].astype('int')\n\n                filenaam='subm_'+clunm+'_'+clanm+'.csv'\n                submit.to_csv(path_or_buf =filenaam, index=False)\n                \n            print(clanm,'0 classifier time',datetime.datetime.now()-start)\n            \n    if len(dtest)==0:       \n        print(pd.DataFrame(results).sort_values(3))\n        submit=[]\n    return submit\n\n#Custom Transformer that extracts columns passed as argument to its constructor \nclass Dummy( ):\n    #Class Constructor \n    def __init__( self, feature_names ):\n        self._feature_names = feature_names \n    \n    #Return self nothing else to do here    \n    def fit( self, X, y = None ):\n        return self \n    \n    #Method that describes what we need this transformer to do\n    def fit_transform( self, X, y = None ):\n        return X \n\nclustertechniques2(train[:10000].drop(['meter_reading'],axis=1).fillna(0).replace(np.inf,0),'EpM2','building_id') #total[len(train):].fillna(0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom sklearn.ensemble import ExtraTreesRegressor\ncla=ExtraTreesRegressor()\ncla.fit(train.drop(['EpM2','meter_reading'],axis=1).fillna(0).replace(np.inf,0).values,train[['EpM2']].fillna(0).values)\npredicts=[]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicts=pd.DataFrame([])\nfor xi in range(4):\n    #fit regression in two chunck !\n    test=loaddata('test',xi,True)\n    test=HotEnc(test,EnergypUse,'leeg','meter_reading',False)\n    preds=test[['row_id','square_feet']]\n    preds['meter']=0.0\n    preds['meter'] = cla.predict(test.drop(['row_id'],axis=1).fillna(0).replace(np.inf,0).values)\n    preds['meter_reading']=np.exp(preds['square_feet'])*preds['meter']\n    predicts=predicts.append(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicts[['row_id','meter_reading']].to_csv('xtree.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}