{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n#         print(dirname, len(filenames))\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import csv","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## adding private key manually"},{"metadata":{"trusted":true},"cell_type":"code","source":"!gcloud auth activate-service-account \\\n  kaggle-automl-vision-sa@trusty-mantra-251121.iam.gserviceaccount.com \\\n          --key-file=/kaggle/input/private-yufengguo556835-gcp-key/key.json \\\n          --project=trusty-mantra-251121","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from google.oauth2 import service_account\n\ncredentials = service_account.Credentials.from_service_account_file('/kaggle/input/private-yufengguo556835-gcp-key/key.json')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PROJECT_ID = 'trusty-mantra-251121'\nREGION='us-central1'           # this MUST be us-central1\nBUCKET_NAME='my-kaggle-data1'  # This bucket should already exist in us-central1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create AutoML Tables Client with manual credentials"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from google.cloud import automl_v1beta1 as automl\nfrom google.cloud import storage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"storage_client = storage.Client(project=PROJECT_ID, credentials=credentials)\ntables_gcs_client = automl.GcsClient(client=storage_client, bucket_name=BUCKET_NAME, credentials=credentials)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"tables_client = automl.TablesClient(project=PROJECT_ID, region=REGION, \n                                    gcs_client=tables_gcs_client, credentials=credentials)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create AutoML Tables client with Kaggle credentials"},{"metadata":{"trusted":true},"cell_type":"code","source":"from kaggle.gcp import KaggleKernelCredentials\nfrom kaggle_secrets import GcpTarget\nimport kaggle_gcp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.environ['KAGGLE_USER_SECRETS_TOKEN']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"KaggleKernelCredentials(GcpTarget.AUTOML).valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"storage_client = storage.Client(project=PROJECT_ID, credentials=KaggleKernelCredentials(GcpTarget.GCS))\ntables_gcs_client = automl.GcsClient(client=storage_client, bucket_name=BUCKET_NAME, credentials=KaggleKernelCredentials(GcpTarget.GCS))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tables_client = automl.TablesClient(project=PROJECT_ID, region=REGION, \n                                    gcs_client=tables_gcs_client, credentials=KaggleKernelCredentials(GcpTarget.AUTOML))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## (WORKAROUND) Create a Tables client with Kaggle credentials"},{"metadata":{"trusted":true},"cell_type":"code","source":"from google.cloud import storage, automl_v1beta1 as automl\n\nstorage_client = storage.Client(project=PROJECT_ID)\ntables_gcs_client = automl.GcsClient(client=storage_client, bucket_name=BUCKET_NAME)\nautoml_client = automl.AutoMlClient()\nprediction_client = automl.PredictionServiceClient()\ntables_client = automl.TablesClient(project=PROJECT_ID, region=REGION, \n                                    client=automl_client, gcs_client=tables_gcs_client, prediction_client=prediction_client)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dataset_metadata = {\"primary_table_spec_id\": \"label\"} \n# # Set dataset name and metadata of the dataset.\n# my_dataset = {\n#     \"tables_dataset_metadata\": dataset_metadata,\n# }\n\ndataset_display_name='kaggle_data_tester'\nmy_dataset = tables_client.create_dataset(dataset_display_name=dataset_display_name)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### List datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"list_datasets = tables_client.list_datasets()\n# Using a dict is problematic since display_name does not have to be unique\n# datasets = { dataset.display_name: dataset.name for dataset in list_datasets }\ndatasets = [ (dataset.display_name, dataset.name) for dataset in list_datasets ]\n\ndatasets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Upload file to GCS"},{"metadata":{"trusted":true},"cell_type":"code","source":"def upload_blob(source_file_name, destination_blob_name, bucket_name=BUCKET_NAME):\n    bucket = storage_client.get_bucket(bucket_name)\n    blob = bucket.blob(destination_blob_name)\n    blob.upload_from_filename(source_file_name)\n    destination_gcs_uri = f'gs://{bucket_name}/{destination_blob_name}'\n    print('File {} uploaded to {}.'.format(\n        source_file_name,\n        destination_gcs_uri))\n    return destination_gcs_uri","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gcs_upload_uri = upload_blob('/kaggle/input/Kannada-MNIST/train.csv', 'kannada_train.csv')\ngcs_upload_uri","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### upload data to AutoML Tables"},{"metadata":{"trusted":true},"cell_type":"code","source":"import_data_operation = tables_client.import_data(\n    dataset=my_dataset,\n    gcs_input_uris=gcs_upload_uri\n)\nprint('Dataset import operation: {}'.format(import_data_operation))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Synchronous check of operation status. Wait until import is done.\nimport_data_operation.result()\ndataset = tables_client.get_dataset(dataset_name=my_dataset.name)\ndataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Normal automl client, Auth with Kaggle Key"},{"metadata":{"trusted":true},"cell_type":"code","source":"automl_client = automl.AutoMlClient()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"project_location = automl_client.location_path(PROJECT_ID, REGION)\n# Specify the image classification type for the dataset, MULTICLASS or MULTILABEL\ndataset_metadata = {\"primary_table_spec_id\": \"label\"} \n# Set dataset name and metadata of the dataset.\nmy_dataset = {\n    \"display_name\": 'kannada_kaggle',\n    \"tables_dataset_metadata\": dataset_metadata,\n}\n\n# Create a dataset with the dataset metadata in the region.\n# dataset = automl_client.create_dataset(project_location, my_dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Automl base client, with manual key"},{"metadata":{"trusted":true},"cell_type":"code","source":"automl_client = automl.AutoMlClient(credentials=credentials)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"project_location = automl_client.location_path(PROJECT_ID, REGION)\n# Specify the image classification type for the dataset, MULTICLASS or MULTILABEL\ndataset_metadata = {\"primary_table_spec_id\": \"label\"} \n# Set dataset name and metadata of the dataset.\nmy_dataset = {\n    \"display_name\": 'kannadakaggle_manual',\n    \"tables_dataset_metadata\": dataset_metadata,\n}\n\n# Create a dataset with the dataset metadata in the region.\n# dataset = automl_client.create_dataset(project_location, my_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train_file = \"../input/Kannada-MNIST/train.csv\"\ndata_test_file = \"../input/Kannada-MNIST/test.csv\" # test file for submission\ndata_dig_file = \"../input/Kannada-MNIST/Dig-MNIST.csv\" #labeled validation set\n\ndf_train = pd.read_csv(data_train_file)\ndf_test = pd.read_csv(data_test_file)\ndf_dig = pd.read_csv(data_dig_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}