Cassandra_PAV_BHU_JEE.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"## Cassandra Round - 1 ##\n## Team Name: PAV_BHU_JEE ##\n\n","metadata":{"id":"HLF1h0go1JFF","execution":{"iopub.status.busy":"2022-04-10T21:25:15.016320Z","iopub.execute_input":"2022-04-10T21:25:15.016535Z","iopub.status.idle":"2022-04-10T21:25:15.023741Z","shell.execute_reply.started":"2022-04-10T21:25:15.016509Z","shell.execute_reply":"2022-04-10T21:25:15.022815Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"# Importing essential libraries\n\nimport pandas as pd \nimport numpy as np \npd.plotting.register_matplotlib_converters()\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-04-10T21:25:15.025713Z","iopub.execute_input":"2022-04-10T21:25:15.026696Z","iopub.status.idle":"2022-04-10T21:25:15.037241Z","shell.execute_reply.started":"2022-04-10T21:25:15.026603Z","shell.execute_reply":"2022-04-10T21:25:15.036333Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"# Loading Training and test datasets\n\ntraining_data = pd.read_csv(\"../input/cassandra-udyam-data-ps-2/trainData (1).csv\")#,index_col=\"id\"\ntest_data = pd.read_csv(\"../input/cassandra-udyam-data-ps-2/testData.csv\")\nX = training_data.copy()\n\n# print(X.columns)\n# print(X.describe())","metadata":{"id":"Ho4kvgoL1fax","outputId":"a7eef88c-7839-47a5-c658-f2319d899140","execution":{"iopub.status.busy":"2022-04-10T21:25:15.040631Z","iopub.execute_input":"2022-04-10T21:25:15.040985Z","iopub.status.idle":"2022-04-10T21:25:15.074654Z","shell.execute_reply.started":"2022-04-10T21:25:15.040947Z","shell.execute_reply":"2022-04-10T21:25:15.073743Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"# converting to proper pandas datetime type\n\nX['Created'] = pd.to_datetime(X['Created'])\nX['Invoice_Date'] = pd.to_datetime(X['Invoice_Date'])\nX['Due_Date'] = pd.to_datetime(X['Due_Date'])\n\n# print(X.columns)\n\n# Extracting 'Day of the month','month','year','hours','minutes',into seperate columns,\n# so as to analyze their individual relationships with target variable.\n\nX['Created Day']=X['Created'].dt.day\nX['Created Month']=X['Created'].dt.month\nX['Created Year']=X['Created'].dt.year\nX['Created Hour']=X['Created'].dt.hour\nX['Created Minutes']=X['Created'].dt.minute\n\nX['Invoice Day']=X['Invoice_Date'].dt.day\nX['Invoice Month']=X['Invoice_Date'].dt.month\nX['Invoice Year']=X['Invoice_Date'].dt.year\n\nX['Due Day']=X['Due_Date'].dt.day\nX['Due Month']=X['Due_Date'].dt.month\nX['Due Year']=X['Due_Date'].dt.year\n\nX['diff_created'] = X['Due_Date'] - X['Created']\nX['diff_created'] = X['diff_created']/np.timedelta64(1,'D')\n# sns.distplot(a=X['diff_created'], kde=False)\n# plt.show()\n# sns.kdeplot(data=X['diff_created'], shade=True)\n# plt.show()\n\nX['diff_due'] = X['Due_Date'] - X['Invoice_Date']\nX['diff_due'] = X['diff_due']/np.timedelta64(1,'D')\n# sns.distplot(a=X['diff_due'], kde=False)\n# plt.show()\n# sns.kdeplot(data=X['diff_due'], shade=True)\n# plt.show()","metadata":{"id":"CCLjHPNF1rFr","outputId":"aaae2aff-d755-4b8e-ae40-6d3ab3dd7b7e","execution":{"iopub.status.busy":"2022-04-10T21:25:15.075916Z","iopub.execute_input":"2022-04-10T21:25:15.077317Z","iopub.status.idle":"2022-04-10T21:25:16.019114Z","shell.execute_reply.started":"2022-04-10T21:25:15.077259Z","shell.execute_reply":"2022-04-10T21:25:16.018377Z"},"trusted":true},"execution_count":26,"outputs":[]},{"cell_type":"code","source":"# checking if the difference columns were added or not \nprint(X.columns)","metadata":{"id":"UKYBEOxI8BQ5","outputId":"4da2ea5d-e605-4776-e7e8-b33debebaabd","execution":{"iopub.status.busy":"2022-04-10T21:25:16.020144Z","iopub.execute_input":"2022-04-10T21:25:16.020348Z","iopub.status.idle":"2022-04-10T21:25:16.025225Z","shell.execute_reply.started":"2022-04-10T21:25:16.020324Z","shell.execute_reply":"2022-04-10T21:25:16.024433Z"},"trusted":true},"execution_count":27,"outputs":[]},{"cell_type":"code","source":"# Frequency encoding Description column\n\nDescription_map = X.groupby(['Description'])['Description'].count()\n# print(Description_map)\n\nX[\"Description_encoded\"] = X['Description'].map(Description_map)\n# print(X.columns)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T21:25:16.026441Z","iopub.execute_input":"2022-04-10T21:25:16.026670Z","iopub.status.idle":"2022-04-10T21:25:16.040208Z","shell.execute_reply.started":"2022-04-10T21:25:16.026643Z","shell.execute_reply":"2022-04-10T21:25:16.039294Z"},"trusted":true},"execution_count":28,"outputs":[]},{"cell_type":"code","source":"# Obtaining pandas profiling report which gives information about missing values,\n# distinct values, mean, mode, categorical or continous, correlation between fetures etc.\n\n! pip install https://github.com/pandas-profiling/pandas-profiling/archive/master.zip\nfrom pandas_profiling import ProfileReport\n\nprofile = ProfileReport(X,title='Pandas Profiling Report',html={'style':{'full_width':True}})\n\nprofile.to_notebook_iframe()","metadata":{"execution":{"iopub.status.busy":"2022-04-10T21:25:16.041799Z","iopub.execute_input":"2022-04-10T21:25:16.042144Z","iopub.status.idle":"2022-04-10T21:26:50.991166Z","shell.execute_reply.started":"2022-04-10T21:25:16.042085Z","shell.execute_reply":"2022-04-10T21:26:50.989657Z"},"trusted":true},"execution_count":29,"outputs":[]},{"cell_type":"code","source":"# Seperating the target variable from from dataset.\n\ny = X['Number_of_Days_until_Payment']\nX.drop(['Number_of_Days_until_Payment'], axis=1, inplace=True)\n# print(y.head())\n# print(X.head())\nprint(X.columns)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T21:26:50.992757Z","iopub.execute_input":"2022-04-10T21:26:50.993308Z","iopub.status.idle":"2022-04-10T21:26:51.003697Z","shell.execute_reply.started":"2022-04-10T21:26:50.993258Z","shell.execute_reply":"2022-04-10T21:26:51.003067Z"},"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"code","source":"# Getting the mutual information scores between various features and target variable.\n# MI gives the degree of dependence between features.\n\nfrom sklearn.feature_selection import mutual_info_regression\n\nX_copy = X.drop(['Vendor_Name','Description','Created','Invoice_Date', 'Due_Date','Description_encoded'], axis=1)\ny_copy = y.copy()\n\n# Label encoding for categoricals\nfor colname in X_copy.select_dtypes(\"object\"):\n    X_copy[colname], _ = X_copy[colname].factorize()\n\n# All discrete features should now have integer dtypes (double-check this before using MI!)\ndiscrete_features = X_copy.dtypes == int\n\n# print(discrete_features)\n\ndef make_mi_scores(X_copy, y_copy, discrete_features):\n    mi_scores = mutual_info_regression(X_copy, y_copy,discrete_features=discrete_features)\n    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X_copy.columns)\n    mi_scores = mi_scores.sort_values(ascending=False)\n    return mi_scores\n\nmi_scores = make_mi_scores(X_copy, y_copy, discrete_features)\nmi_scores[::3]\nprint(mi_scores)\n\ndef plot_mi_scores(scores):\n    scores = scores.sort_values(ascending=True)\n    width = np.arange(len(scores))\n    ticks = list(scores.index)\n    plt.barh(width, scores)\n    plt.yticks(width, ticks)\n    plt.title(\"Mutual Information Scores\")\n\n\nplt.figure(dpi=100, figsize=(8, 5))\nplot_mi_scores(mi_scores)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T21:26:51.005000Z","iopub.execute_input":"2022-04-10T21:26:51.005480Z","iopub.status.idle":"2022-04-10T21:26:52.221964Z","shell.execute_reply.started":"2022-04-10T21:26:51.005438Z","shell.execute_reply":"2022-04-10T21:26:52.221133Z"},"trusted":true},"execution_count":31,"outputs":[]},{"cell_type":"code","source":"# Dropping the redundand columns\n\nX.drop(['Description','Created','Invoice_Date', 'Due_Date','Created Minutes'], axis=1, inplace=True)\n\nfrom sklearn.model_selection import train_test_split\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2,random_state=0)","metadata":{"id":"VU1ofAD_EMCG","execution":{"iopub.status.busy":"2022-04-10T21:26:52.223355Z","iopub.execute_input":"2022-04-10T21:26:52.223684Z","iopub.status.idle":"2022-04-10T21:26:52.232840Z","shell.execute_reply.started":"2022-04-10T21:26:52.223652Z","shell.execute_reply":"2022-04-10T21:26:52.231926Z"},"trusted":true},"execution_count":32,"outputs":[]},{"cell_type":"code","source":"# Target encoding the Vendor_Name columns.\n\nfrom category_encoders import MEstimateEncoder\nencoder = MEstimateEncoder(cols=[\"Vendor_Name\"], m=5.0)\n# print(X_train.columns)\nencoder.fit(X_valid,y_valid)\nX_train = encoder.transform(X_train)\n# X_valid = encoder.transform(X_valid)\n# print(X_train.columns)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T21:26:52.233944Z","iopub.execute_input":"2022-04-10T21:26:52.234237Z","iopub.status.idle":"2022-04-10T21:26:52.263558Z","shell.execute_reply.started":"2022-04-10T21:26:52.234197Z","shell.execute_reply":"2022-04-10T21:26:52.262826Z"},"trusted":true},"execution_count":33,"outputs":[]},{"cell_type":"code","source":"# Normalized training data,since I wanted to perform k-Means clustering.\n\nsubracter = X_train.mean(axis=0)\ndivider =  X_train.std(axis=0)\nX_train = (X_train - subracter)/(divider)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T21:26:52.264826Z","iopub.execute_input":"2022-04-10T21:26:52.265057Z","iopub.status.idle":"2022-04-10T21:26:52.274953Z","shell.execute_reply.started":"2022-04-10T21:26:52.265033Z","shell.execute_reply":"2022-04-10T21:26:52.274153Z"},"trusted":true},"execution_count":34,"outputs":[]},{"cell_type":"code","source":"# Performing K-Means Clustering. This was done just 10 mins before deadline, Hence could not be optimized.\n\nfrom sklearn.cluster import KMeans\n\n\nkmeans = KMeans(n_clusters=10, n_init=10, random_state=0)\nX_train[\"Cluster\"] = kmeans.fit_predict(X_train[['Vendor_Name', 'Amount','diff_created', 'diff_due']])","metadata":{"execution":{"iopub.status.busy":"2022-04-10T21:26:52.276461Z","iopub.execute_input":"2022-04-10T21:26:52.276831Z","iopub.status.idle":"2022-04-10T21:26:55.281339Z","shell.execute_reply.started":"2022-04-10T21:26:52.276799Z","shell.execute_reply":"2022-04-10T21:26:55.280549Z"},"trusted":true},"execution_count":35,"outputs":[]},{"cell_type":"code","source":"# Importing Catboost and MSE(Evaluation metric).\n\n!pip install catboost\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_squared_error","metadata":{"id":"9la_eRRcGyy6","execution":{"iopub.status.busy":"2022-04-10T21:26:55.282426Z","iopub.execute_input":"2022-04-10T21:26:55.282823Z","iopub.status.idle":"2022-04-10T21:27:02.599223Z","shell.execute_reply.started":"2022-04-10T21:26:55.282781Z","shell.execute_reply":"2022-04-10T21:27:02.598015Z"},"trusted":true},"execution_count":36,"outputs":[]},{"cell_type":"code","source":"# Training the model.\n# I didn't use validation data since I had already used it for target encoding.\n\nmy_model = CatBoostRegressor(n_estimators=600,learning_rate=0.05)\nmy_model.fit(X_train,y_train,verbose=False)\nprint(my_model.best_score_)\nprint(\"Best iteration :\",my_model.best_iteration_)","metadata":{"id":"fnOprhdoHEMg","outputId":"b5472254-820b-495c-f4e6-52afb455a1c3","execution":{"iopub.status.busy":"2022-04-10T21:27:02.601521Z","iopub.execute_input":"2022-04-10T21:27:02.601832Z","iopub.status.idle":"2022-04-10T21:27:03.963405Z","shell.execute_reply.started":"2022-04-10T21:27:02.601788Z","shell.execute_reply":"2022-04-10T21:27:03.962723Z"},"trusted":true},"execution_count":37,"outputs":[]},{"cell_type":"code","source":"# Printing the model parameters.\nprint(my_model.get_all_params())","metadata":{"execution":{"iopub.status.busy":"2022-04-10T21:27:03.964325Z","iopub.execute_input":"2022-04-10T21:27:03.964689Z","iopub.status.idle":"2022-04-10T21:27:03.970091Z","shell.execute_reply.started":"2022-04-10T21:27:03.964657Z","shell.execute_reply":"2022-04-10T21:27:03.969254Z"},"trusted":true},"execution_count":38,"outputs":[]},{"cell_type":"code","source":"# Preprocessing the test data.\n\n# print(test_data.head())\nX_test = test_data.copy()\nX_test['Created'] = pd.to_datetime(X_test['Created'])\nX_test['Invoice_Date'] = pd.to_datetime(X_test['Invoice_Date'])\nX_test['Due_Date'] = pd.to_datetime(X_test['Due_Date'])\n\nX_test['Created Day']=X_test['Created'].dt.day\nX_test['Created Month']=X_test['Created'].dt.month\nX_test['Created Year']=X_test['Created'].dt.year\nX_test['Created Hour']=X_test['Created'].dt.hour\nX_test['Created Minutes']=X_test['Created'].dt.minute\n\nX_test['Invoice Day']=X_test['Invoice_Date'].dt.day\nX_test['Invoice Month']=X_test['Invoice_Date'].dt.month\nX_test['Invoice Year']=X_test['Invoice_Date'].dt.year\n\nX_test['Due Day']=X_test['Due_Date'].dt.day\nX_test['Due Month']=X_test['Due_Date'].dt.month\nX_test['Due Year']=X_test['Due_Date'].dt.year\n\nX_test['diff_created'] = X_test['Due_Date'] - X_test['Created']\nX_test['diff_created'] = X_test['diff_created']/np.timedelta64(1,'D')\n# sns.distplot(a=X['diff_created'], kde=False)\n# plt.show()\n# sns.kdeplot(data=X['diff_created'], shade=True)\n# plt.show()\n\nX_test['diff_due'] = X_test['Due_Date'] - X_test['Invoice_Date']\nX_test['diff_due'] = X_test['diff_due']/np.timedelta64(1,'D')\n\n# X_test['Description_null'] = X_test['Description'].isnull()\n# X_test['Description_notnull'] = X_test['Description'].notnull()\n\nX_test[\"Description_encoded\"]= X_test['Description'].map(Description_map)\nX_test.drop(['Description','Created', 'Invoice_Date', 'Due_Date','Created Minutes'], axis=1, inplace=True)\nX_test = encoder.transform(X_test)\n\nX_test =  (X_test - subracter) / divider\n\nX_test['Cluster'] = kmeans.predict(X_test[['Vendor_Name', 'Amount','diff_created', 'diff_due']])\nprint(X_test.columns)\n# print(X_test.head())","metadata":{"id":"rdnqK89wHPJf","outputId":"2bc82c7d-8a9c-4a0a-ce80-0b65126ea7c8","execution":{"iopub.status.busy":"2022-04-10T21:27:03.971140Z","iopub.execute_input":"2022-04-10T21:27:03.971343Z","iopub.status.idle":"2022-04-10T21:27:04.333214Z","shell.execute_reply.started":"2022-04-10T21:27:03.971318Z","shell.execute_reply":"2022-04-10T21:27:04.332268Z"},"trusted":true},"execution_count":39,"outputs":[]},{"cell_type":"code","source":"# my_model_2 =  CatBoostRegressor(n_estimators=50,learning_rate=0.05)\n# my_model_2.fit(X, y,verbose=False)\n# print(my_model_2.best_score_)\n# print(\"Best iteration :\",my_model_2.best_iteration_)","metadata":{"id":"CBaw8Z57LAU1","outputId":"62961dd0-e0da-449e-d3fe-ad6ac54addff","execution":{"iopub.status.busy":"2022-04-10T21:27:04.334928Z","iopub.execute_input":"2022-04-10T21:27:04.335283Z","iopub.status.idle":"2022-04-10T21:27:04.341060Z","shell.execute_reply.started":"2022-04-10T21:27:04.335238Z","shell.execute_reply":"2022-04-10T21:27:04.340238Z"},"trusted":true},"execution_count":40,"outputs":[]},{"cell_type":"code","source":"# Creating and saving test predictions.\n\npredictions = my_model.predict(X_test)\noutput = pd.DataFrame({'Vendor_Name': test_data['Vendor_Name'],\n                       'Number_of_Days_Until_Payment': predictions})\noutput.to_csv('submission1.csv', index=False)","metadata":{"id":"ZjzDyU0OH-MQ","execution":{"iopub.status.busy":"2022-04-10T21:27:04.342506Z","iopub.execute_input":"2022-04-10T21:27:04.343089Z","iopub.status.idle":"2022-04-10T21:27:04.395375Z","shell.execute_reply.started":"2022-04-10T21:27:04.343044Z","shell.execute_reply":"2022-04-10T21:27:04.394258Z"},"trusted":true},"execution_count":41,"outputs":[]}]}