{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* 파일 불러와서 데이터 확인","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/facebook-recruiting-iv-human-or-bot/train.csv.zip')\ntrain.head(20)\n# 로봇이 보안이 취약한 은행에 접근할 수도","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.nunique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/facebook-recruiting-iv-human-or-bot/test.csv.zip')\ntest.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = pd.concat([train,test])     \nall_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bid = pd.read_csv('/kaggle/input/facebook-recruiting-iv-human-or-bot/bids.csv.zip')\nbid.head()\n# ip는 새로운 정보가 많이 없어서 오히려 과대적합성 있어보임\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bid['bidder_id'].value_counts()   # 구매횟수로 로봇 예측 가능\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* bid 파일에서 필요한 정보들 가져와서 all_data 파일에 합치기","metadata":{}},{"cell_type":"code","source":"a = bid.groupby(\"bidder_id\")[\"bid_id\"].count().reset_index() # 구매 횟수\na\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b = bid.groupby('bidder_id')['auction','country','device'].nunique().reset_index()        # 옥션 사이트, 접속 나라, 접속 기기 갯수\nb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c = b.merge(a,how='left')\nall_data = all_data.merge(c, how='left') \nall_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* boxplot으로 의미있는 자료인지 확인","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nall_data_=all_data[all_data['bid_id']<20000]\n\nsns.boxplot(all_data['outcome'],all_data_['bid_id'])\nplt.show() \nsns.boxplot(all_data['outcome'],all_data['auction'])\nplt.show() \nsns.boxplot(all_data['outcome'],all_data['country'])\nplt.show() \nsns.boxplot(all_data['outcome'],all_data['device'])\nplt.show() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* datetime 과 time 활용해서 정보 얻기","metadata":{"trusted":true}},{"cell_type":"code","source":"from datetime import datetime\nbid['datetime'] = bid['time'].apply(lambda x : datetime.fromtimestamp(x/7000000)) # 숫자는 적당히 잘라서 찾아냄\nbid['year'] = bid['datetime'].dt.year\nbid['month'] = bid['datetime'].dt.month\nbid['day'] = bid['datetime'].dt.day\nbid.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean = bid.groupby('bidder_id')['year','month','day'].mean().reset_index() # 구매 날짜연/월/일 평균\nmean","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = all_data.merge(mean,how='left')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_diff = bid.groupby('bidder_id')['time'].apply(lambda x : x.max()-x.min()).reset_index() \nall_data = all_data.merge(time_diff,how='left')\nall_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['time'] = all_data['time']/all_data['bid_id'] # 거래 사이의 평균 시간\nall_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* 기존 익명화된 정보 가공","metadata":{}},{"cell_type":"code","source":"all_data['payment_account'] = all_data['payment_account'].apply(lambda x : x[:5]) # 뒷 5자리만 반복됨 -> 은행 등 공통 정보, 보안에 취약한 은행은 로봇 비율이 높을 것 같다\nall_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['payment_account'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['address'].sort_values().head(30)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['address'] = all_data['address'].apply(lambda x : x[:5]) # 마찬가지로 뒷 5자리만 반복 -> 포털 사이트 명\nall_data['address'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* 모델링","metadata":{}},{"cell_type":"code","source":"# 문자를 숫자로 자동으로 바꿔주는 라이브러리 사용\n\nfrom sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\nall_data['payment_account'] = le.fit_transform(all_data['payment_account'])\nall_data['address'] = le.fit_transform(all_data['address'])\nall_data.head()\n\n# 각 카테고리의 문자가 몇번으로 될지 알아서 등록 : fit - lookup table\n# 실제로 바꿈 : transform\n# fit_transform 으로 한번에 가능","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data2 = all_data.drop(['bidder_id', 'outcome'],1)\nall_data2.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data2 = all_data2.fillna(-1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train2 = all_data2[:len(train)]\ntrain2.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test2 = all_data2[len(train):]\ntest2.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nrf = RandomForestClassifier(n_estimators = 170, max_depth = 7)\nrf.fit(train2,train['outcome'])\nresult = rf.predict_proba(test2)\n\n# 데이터의 패턴, 복잡도 확인 - 데이터의 갯수(적음), 칼럼의 갯수(적음), 카테고리 칼럼의 카테고리 갯수(보통) -> 단순한 데이터라고 판단","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = result[:,1]\nresult","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['outcome'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/facebook-recruiting-iv-human-or-bot/sampleSubmission.csv')\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['prediction'] = result\nsub.to_csv('sub.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# private score : 0.92 - 상위 15% ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}