Spark_Template.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMRAB0NazW8vqDmIjWnUovR",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kiat/Cloud-Computing/blob/main/Spark_Template.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "YBUjSye-RKOI"
      },
      "outputs": [],
      "source": [
        "!pip install -qU pyspark findspark"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import findspark\n",
        "findspark.init()"
      ],
      "metadata": {
        "id": "NmPM18WjRdAS"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession\n",
        "\n",
        "spark = SparkSession.builder.appName(\"YourAppName\").getOrCreate()"
      ],
      "metadata": {
        "id": "u7xRS6SrRN12"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark import SparkConf,SparkContext\n",
        "\n",
        "sc = SparkContext.getOrCreate()"
      ],
      "metadata": {
        "id": "HxosgYqAROl_"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rdd1 = sc.parallelize(range(1, 10))\n",
        "rdd1.collect()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "liWHIgGbTMMQ",
        "outputId": "555863f5-164d-49fa-de73-2a80af48e6f0"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 2, 3, 4, 5, 6, 7, 8, 9]"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# An example to load dataset from file.\n",
        "californiaHousing = sc.textFile(\"./sample_data/california_housing_test.csv\")\n",
        "californiaHousing.take(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DABvxWrFTMVZ",
        "outputId": "4336b384-11eb-4bc3-bd90-acf694cbbc85"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['\"longitude\",\"latitude\",\"housing_median_age\",\"total_rooms\",\"total_bedrooms\",\"population\",\"households\",\"median_income\",\"median_house_value\"',\n",
              " '-122.050000,37.370000,27.000000,3885.000000,661.000000,1537.000000,606.000000,6.608500,344700.000000']"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    }
  ]
}