{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install flaml\n!pip install pyspark\n!pip install synapseml\n!pip install scrapbook\n!pip install recommenders","metadata":{"execution":{"iopub.status.busy":"2022-07-07T04:18:44.403419Z","iopub.execute_input":"2022-07-07T04:18:44.403842Z","iopub.status.idle":"2022-07-07T04:19:51.038038Z","shell.execute_reply.started":"2022-07-07T04:18:44.403808Z","shell.execute_reply":"2022-07-07T04:19:51.03679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport pyspark\nfrom pyspark.ml import PipelineModel\nfrom pyspark.ml.feature import FeatureHasher\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nimport papermill as pm\nimport scrapbook as sb\n\nfrom recommenders.utils.notebook_utils import is_databricks\nfrom recommenders.utils.spark_utils import start_or_get_spark\nfrom recommenders.datasets.criteo import load_spark_df\nfrom recommenders.datasets.spark_splitters import spark_random_split\n\nfrom pyspark.ml.feature import VectorAssembler, StandardScaler, OneHotEncoder, StringIndexer\nfrom recommenders.utils.spark_utils import MMLSPARK_REPO, MMLSPARK_PACKAGE\npackages = [MMLSPARK_PACKAGE]\nrepos = [MMLSPARK_REPO]\nspark = start_or_get_spark(packages=packages, repositories=repos)\ndbutils = None\n\nfrom flaml import AutoML\nimport pandas as pd\nimport numpy as np\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T04:50:15.825703Z","iopub.execute_input":"2022-07-07T04:50:15.826148Z","iopub.status.idle":"2022-07-07T04:50:15.84092Z","shell.execute_reply.started":"2022-07-07T04:50:15.826117Z","shell.execute_reply":"2022-07-07T04:50:15.839552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nautoml = AutoML()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T04:21:47.799954Z","iopub.execute_input":"2022-07-07T04:21:47.800378Z","iopub.status.idle":"2022-07-07T04:21:47.807179Z","shell.execute_reply.started":"2022-07-07T04:21:47.800329Z","shell.execute_reply":"2022-07-07T04:21:47.805398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# File location and type\nfile_location =  \"../input/anomaly-sub/train_sub.csv\"\nfile_type = \"csv\"\n\n# CSV options\ninfer_schema = \"true\"\nfirst_row_is_header = \"true\"\ndelimiter = \",\"\n\n# The applied options are for CSV files. For other file types, these will be ignored.\ndf = spark.read.format(file_type) \\\n  .option(\"inferSchema\", infer_schema) \\\n  .option(\"header\", first_row_is_header) \\\n  .option(\"sep\", delimiter) \\\n  .load(file_location)\ndf.limit(2).toPandas().head()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:00:57.212563Z","iopub.execute_input":"2022-07-07T05:00:57.213004Z","iopub.status.idle":"2022-07-07T05:01:02.209936Z","shell.execute_reply.started":"2022-07-07T05:00:57.21297Z","shell.execute_reply":"2022-07-07T05:01:02.208604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = df.select([c for c in df.columns if c in ['meter_reading','month_x','actual_anomaly']])\nFeatures_col = df_sub.schema.names[0:-1]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:02:29.770319Z","iopub.execute_input":"2022-07-07T05:02:29.770729Z","iopub.status.idle":"2022-07-07T05:02:29.793961Z","shell.execute_reply.started":"2022-07-07T05:02:29.770696Z","shell.execute_reply":"2022-07-07T05:02:29.793116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# File location and type\nfile_location =  \"../input/anomaly-sub/test_sub50_1.csv\"\nfile_type = \"csv\"\n\n# CSV options\ninfer_schema = \"true\"\nfirst_row_is_header = \"true\"\ndelimiter = \",\"\n\n# The applied options are for CSV files. For other file types, these will be ignored.\ndf_test = spark.read.format(file_type) \\\n  .option(\"inferSchema\", infer_schema) \\\n  .option(\"header\", first_row_is_header) \\\n  .option(\"sep\", delimiter) \\\n  .load(file_location)\ndf_test.limit(2).toPandas().head()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:00:28.484542Z","iopub.execute_input":"2022-07-07T05:00:28.484934Z","iopub.status.idle":"2022-07-07T05:00:31.319024Z","shell.execute_reply.started":"2022-07-07T05:00:28.484902Z","shell.execute_reply":"2022-07-07T05:00:31.317809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.select([c for c in df_test.columns if c in ['meter_reading','month_x']]).show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:01:43.091741Z","iopub.execute_input":"2022-07-07T05:01:43.092134Z","iopub.status.idle":"2022-07-07T05:01:43.184358Z","shell.execute_reply.started":"2022-07-07T05:01:43.092104Z","shell.execute_reply":"2022-07-07T05:01:43.183124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nvectorAssembler = VectorAssembler(inputCols = Features_col, outputCol = 'features')\n#vectorAssembler.setHandleInvalid(\"skip\").transform(df_sub).show\ntrain_sprk = vectorAssembler.transform(df_sub)\ntest_sprk = vectorAssembler.transform(df_test)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.datasets import make_classification\ny = train_sprk[\"actual_anomaly\"]\nX = train_sprk[Features_col]\nX, y = make_classification()\nX_train, X_test, y_train, y_test = train_test_split(X, y)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:06:41.796793Z","iopub.execute_input":"2022-07-07T05:06:41.797232Z","iopub.status.idle":"2022-07-07T05:06:41.884102Z","shell.execute_reply.started":"2022-07-07T05:06:41.797198Z","shell.execute_reply":"2022-07-07T05:06:41.883236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from flaml import logger\nimport logging\nlogger.setLevel(logging.WARNING)\n\nsettings = {\n    \"time_budget\": 60,  # total running time in seconds\n    \"metric\": 'roc_auc',  \n    \"estimator_list\": ['xgboost'],                    \n    \"task\": 'classification',  # task type\n    \"log_file_name\": 'experiment.log',  # flaml log file\n    \"seed\": 76,\n    \"verbose\" : 0\n  \n    \n   \n            # random seed 22  786\n}\n\nautoml.fit(X_train=X_train, y_train=y_train, **settings)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:04:34.158384Z","iopub.execute_input":"2022-07-07T05:04:34.158837Z","iopub.status.idle":"2022-07-07T05:05:34.173391Z","shell.execute_reply.started":"2022-07-07T05:04:34.158801Z","shell.execute_reply":"2022-07-07T05:05:34.172078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''retrieve best config and best learner'''\nprint('Best ML leaner:', automl.best_estimator)\nprint('Best hyperparmeter config:', automl.best_config)\nprint('Best accuracy on validation data: {0:.4g}'.format(1-automl.best_loss))\nprint('Training duration of best run: {0:.4g} s'.format(automl.best_config_train_time))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:05:45.901402Z","iopub.execute_input":"2022-07-07T05:05:45.901854Z","iopub.status.idle":"2022-07-07T05:05:45.910103Z","shell.execute_reply.started":"2022-07-07T05:05:45.901807Z","shell.execute_reply":"2022-07-07T05:05:45.908796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''compute predictions of testing dataset''' \ny_pred = automl.predict(X_test)\nprint('Predicted labels', y_pred)\nprint('True labels', y_test)\ny_pred_proba = automl.predict_proba(X_test)[:,1]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:05:52.260517Z","iopub.execute_input":"2022-07-07T05:05:52.260984Z","iopub.status.idle":"2022-07-07T05:05:52.271619Z","shell.execute_reply.started":"2022-07-07T05:05:52.260949Z","shell.execute_reply":"2022-07-07T05:05:52.270549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  prediction on test data\ny_pred = automl.predict(test_sprk)\nprint('Predicted labels', y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T05:06:46.972023Z","iopub.execute_input":"2022-07-07T05:06:46.972414Z","iopub.status.idle":"2022-07-07T05:06:47.029194Z","shell.execute_reply.started":"2022-07-07T05:06:46.972381Z","shell.execute_reply":"2022-07-07T05:06:47.027484Z"},"trusted":true},"execution_count":null,"outputs":[]}]}