spark/salesdata/FireIncident.ipynb

Fire Incident Data Preparation.ipynb

{
  "cells": [
    {
      "metadata": {},
      "id": "860dac6b",
      "cell_type": "markdown",
      "source": "# Fire Incident\n"
    },
    {
      "metadata": {
        "trusted": false
      },
      "id": "0887d943",
      "cell_type": "code",
      "source": "import findspark\nfindspark.init('/Users/DOU2274/spark/spark-3.1.1-bin-hadoop2.7')\nfrom pyspark.sql import SparkSession\nspark = (SparkSession.builder.appName(\"project\").getOrCreate())\n",
      "execution_count": 19,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": false
      },
      "id": "d9a2b47e",
      "cell_type": "code",
      "source": "file = '/Users/DOU2274/spark/salesdata/fire-incidents.csv'\n\nfire_df = (spark.read.format(\"csv\")\n           .option(\"header\", \"true\")\n           .option(\"inferSchema\", \"true\")\n           .load(file)\n           )\nfire_df.select(\"IncidentNumber\", \"IncidentDate\", \"city\").show(5, truncate=False)\n\n#fire_df.schema",
      "execution_count": 16,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "+--------------+-------------------+-------------+\n|IncidentNumber|IncidentDate       |city         |\n+--------------+-------------------+-------------+\n|20104668      |2020-09-11 00:00:00|San Francisco|\n|20104708      |2020-09-11 00:00:00|San Francisco|\n|20104648      |2020-09-10 00:00:00|San Francisco|\n|20104598      |2020-09-10 00:00:00|San Francisco|\n|20104575      |2020-09-10 00:00:00|San Francisco|\n+--------------+-------------------+-------------+\nonly showing top 5 rows\n\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": false
      },
      "id": "ab74505d",
      "cell_type": "code",
      "source": "output = '/Users/DOU2274/spark/salesdata/output/fireincidents'\nfire_df.write.format(\"parquet\").mode(\"overwrite\").save(output)",
      "execution_count": 17,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": false
      },
      "id": "2a725c01",
      "cell_type": "code",
      "source": "file = '/Users/DOU2274/spark/salesdata/fire-incidents.csv'\n\nfire_df = (spark.read.csv(file, inferSchema=True, header=True))\nfire_df.select(\"IncidentNumber\", \"IncidentDate\", \"city\").show(5, truncate=False)\n\n#fire_df.schema",
      "execution_count": 18,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": "+--------------+-------------------+-------------+\n|IncidentNumber|IncidentDate       |city         |\n+--------------+-------------------+-------------+\n|20104668      |2020-09-11 00:00:00|San Francisco|\n|20104708      |2020-09-11 00:00:00|San Francisco|\n|20104648      |2020-09-10 00:00:00|San Francisco|\n|20104598      |2020-09-10 00:00:00|San Francisco|\n|20104575      |2020-09-10 00:00:00|San Francisco|\n+--------------+-------------------+-------------+\nonly showing top 5 rows\n\n"
        }
      ]
    },
    {
      "metadata": {
        "trusted": false
      },
      "id": "0a7f2f54",
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3 (ipykernel)",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.9.5",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "gist": {
      "id": "25ab55c8183f0b590a77c9f94a59bbec",
      "data": {
        "description": "spark/salesdata/FireIncident.ipynb",
        "public": true
      }
    },
    "_draft": {
      "nbviewer_url": "https://gist.github.com/25ab55c8183f0b590a77c9f94a59bbec"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}

Untitled.ipynb