{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#load necessary libraries\nimport urllib.request\nfrom bs4 import BeautifulSoup\nimport requests\n#URLs can be fetched using the Python module urllib.request.\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \ndef describepd(data):\n    output=[]\n    for li in data.columns:\n        aantal=len(data)\n        vul=len(data[li].dropna())\n        vtyp=data[li].dtypes\n        uniek=len(data[li].unique())\n        if uniek==aantal:\n            veldindex='indexfield'\n        elif uniek==1:\n            veldindex='constant'\n        else:\n            veldindex=''\n        output.append([li,vtyp,np.round(100-vul/aantal*100),np.round(uniek/aantal*100),uniek,veldindex] )\n    return pd.DataFrame(output,columns=['label','dtype','%empty','%uniek','aantaluniek','keyparam'])        ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# some interestings facts\n\n* there is a relation between publications and competitions, hence Kaggle is a fertile environment for datascience\n* there is a progress in AI tools that can be reflected, as fe \n** Roberta,embeddings for linguistic models, \n** NeuralNetworks for image recognition \n** LGBM gradient boost methods","metadata":{}},{"cell_type":"markdown","source":"# import data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\ndata=pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/kaggle_writeups_20230510.csv\",sep=\",\")\ndescribepd(data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta=pd.read_json('/kaggle/input/2023-kaggle-ai-report/arxiv_metadata_20230510.json',lines=True)\nmeta.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# number of kaggle-competitions versus publications each year\n\n\nmore and more contests lead to scientific publications\non average we discover a ratio of 2 publications per competition","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import stopwords\nstops = list(set(stopwords.words('english')))\nstopsxtr=stops+['model','data','train','training','used','using','kaggle','score','feature','competition','prediction','public','private','thank','lb','https']\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['year']=pd.to_datetime(data['Date of Writeup']).dt.year\ncomp=data.groupby('Title of Competition').max()\n\ncomp.groupby('year').count()[['Competition URL']].plot()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"urls=data['Competition URL'].unique()\nfor ui in urls:\n    print(ui)\n    page = urllib.request.urlopen(ui)\n    soup=BeautifulSoup(page)\n    print (soup.prettify) #can be used to structure the dataimport numpy as np # linear algebra\n    ","metadata":{"execution":{"iopub.status.busy":"2023-05-18T16:24:36.950044Z","iopub.execute_input":"2023-05-18T16:24:36.950470Z","iopub.status.idle":"2023-05-18T16:24:38.749331Z","shell.execute_reply.started":"2023-05-18T16:24:36.950440Z","shell.execute_reply":"2023-05-18T16:24:38.748219Z"},"_kg_hide-input":true}},{"cell_type":"code","source":"meta['year']=pd.to_datetime(meta['update_date']).dt.year\nkagglecount=meta[meta['abstract'].str.lower().str.find('kaggle')>0][['title','abstract','year']]\nkagglecount.groupby('year').count().plot()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# unsupervised: there are 3 clusters developping over time","metadata":{}},{"cell_type":"code","source":"### import re\nimport numpy as np\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import TruncatedSVD\nimport matplotlib.pyplot as plt\n\nfrom io import StringIO\nfrom html.parser import HTMLParser\n\n# clean data\nclass MLStripper(HTMLParser):\n    def __init__(self):\n        super().__init__()\n        self.reset()\n        self.strict = False\n        self.convert_charrefs= True\n        self.text = StringIO()\n    def handle_data(self, d):\n        self.text.write(d)\n    def get_data(self):\n        return self.text.getvalue()\n\ndef strip_tags(html):\n    s = MLStripper()\n    s.feed(html)\n    return s.get_data()\n\n\ndef zoeknr(txt):\n    import re\n    temp=re.findall('[0-9]',txt)\n    if len(temp)==0:\n        temp=np.nan\n    else:\n        temp=temp[0]\n    return temp\n\n\n\ndef showwordcloud(data,maxdist2):\n    from wordcloud import WordCloud\n\n    # Alle teksten per cluster verzamelen\n    cluster_text = {}\n    for i, tekst in data.iloc[maxdist2]['tekst_data'].reset_index().values:\n        cli=data.iloc[i].clusters\n        if cli  not in cluster_text:\n            cluster_text[cli] = \"\"\n        cluster_text[cli] += ' '.join([w for w in tekst.split(' ') if w.lower() not in stopsxtr]) + \" \"\n\n    # Genereren van de word clouds per cluster\n    for cluster, tekst in cluster_text.items():\n        wordcloud = WordCloud().generate(tekst)\n        plt.imshow(wordcloud, interpolation='bilinear')\n        plt.title(f\"Cluster {cluster}\")\n        plt.axis(\"off\")\n        plt.show()\n\ndef argminmax(datmp,axiss,ksize,atype='minmax'):\n    from sklearn.metrics.pairwise import euclidean_distances,manhattan_distances,cosine_distances\n\n    if atype=='circle':\n        datmn=datmp.mean(axis=0).reshape(1,-1)\n        datdist=euclidean_distances(datmp,datmn)\n        datdist=[x[0] for x in datdist]\n        datsrt=np.argsort(datdist,axis=axiss)\n        topres=list(datsrt[-ksize*5:])\n        \n    else:\n        datsrt=np.argsort(datmp[:,0],axis=axiss)\n        topres=[xi for xi in list(datsrt[-ksize*2:])+list(datsrt[:ksize*2])]\n        datsrt=np.argsort(datmp[:,1],axis=axiss)\n        topres=topres+[xi for xi in  list(datsrt[-ksize*2:])+list(datsrt[:ksize*2]) ]\n        \n    return topres\n\n\n\ndef scattergraph(x,y,clusters,title,maxdist2):\n    # Scatter plot van de clusters\n    plt.scatter(x,y, c=clusters, cmap='viridis')\n    # Weergeven van de clusterlabels\n    #for i, label in enumerate(clusters[:100]):\n    #    plt.annotate(label, (x[i], y[i]), textcoords=\"offset points\", xytext=(0,10), ha='center')\n    \n    for xi in maxdist2:\n        plt.annotate(clusters[xi], (x[xi], y[xi]), textcoords=\"offset points\", xytext=(0,10), ha='center')\n        #plt.annotate(clusters[yi], (x[yi], y[yi]), textcoords=\"offset points\", xytext=(0,10), ha='center')\n    # Labels en titel van de grafiek\n    plt.xlabel('svd1')\n    plt.ylabel('svd2')\n    plt.title(title)\n    plt.show()\n    return\n\n","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# HTML-tags verwijderen\ndata['tekst_data'] = data['Writeup'].fillna('').apply(strip_tags)\n\ndata['Nummer']=data['Title of Writeup'].apply(zoeknr)\ndata['year']=[pd.to_datetime(di).year for di in data['Date of Writeup'] ]\n\n# TF-IDF vectorisatie\nvectorizer = TfidfVectorizer(stop_words=stops)\n#tfidf_sci=vectorizer.fit_transform(scitxt['tekst_data'].fillna(''))\ntfidf_matrix = vectorizer.fit_transform(data['tekst_data'] )\n\n# TruncatedSVD voor dimensionaliteitsvermindering\nsvd = TruncatedSVD(n_components=100)\nsvd_matrix = svd.fit_transform(tfidf_matrix)\nprint(svd_matrix.shape)\n# K-means clustering\nk = 3  # Aantal clusters\nkmeans = KMeans(n_clusters=k)\nkmeans.fit(svd_matrix)\n\n# Toewijzing van documenten aan clusters\ndata['clusters'] = kmeans.labels_\n\ndef klust(simdata):\n    from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n    from sklearn.neighbors import KNeighborsClassifier,NeighborhoodComponentsAnalysis\n    from sklearn.decomposition import PCA,TruncatedSVD,NMF,FastICA\n    from umap import UMAP  # knn lookalike of tSNE but faster, so scales up\n    from sklearn.manifold import TSNE,Isomap,SpectralEmbedding,spectral_embedding,LocallyLinearEmbedding,MDS #limit number of records to 100000\n    nummercl=5\n    clustersci = [PCA(n_components=nummercl,random_state=0,whiten=True),\n                  TruncatedSVD(n_components=nummercl),\n                  FastICA(n_components=nummercl)\n                ] \n    for cli in clustersci:\n        print(cli)\n        clunm=str(cli)[:10]\n        if clunm=='NMF':\n            simdata=simdata-simdata.min()+1\n        svd_matrix2 = cli.fit_transform(simdata)\n        maxdist=argminmax(svd_matrix2[:,:2],0,10)\n        scattergraph(svd_matrix2[:, 0], svd_matrix2[:, 1],data['clusters'],clunm,maxdist)\n        showwordcloud(data,maxdist)\n    return\n#scattergraph(svd_matrix[:, 0], svd_matrix[:, 1],data['year'])\nklust(svd.fit_transform( tfidf_matrix))#.dot(tfidf_sci.T) )  )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# evolution of the clusters over the years\n\n\n* 1 cluster2010: epoch, cross validation, lgb,lightgbm,NN,Catboost, stacking\n* 2 cluster2016: image model, segmentation, augmentation, cross validation\n* 0 cluster2019: Deberta V2, V3, roberta, ensemble, sentence embedding, text, pseudo label , fine tuning feature extraction","metadata":{}},{"cell_type":"code","source":"data[['year','clusters','Writeup']].groupby(['year','clusters']).count().reset_index().pivot('year','clusters','Writeup').plot()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/2023-kaggle-ai-report/sample_submission.csv\")\nsubmission.loc[0]['value'] = 'Text data'\nsubmission.loc[1]['value'] = 'https://www.kaggle.com/code/plarmuseau/grab-the-competitiondata'\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}