Geocoding Setup using Selenium

extract_from_geojson.py

import json
from shapely.geometry import shape, Point

with open('./Data/nyc-neighborhoods.geo.json') as f:
    js = json.load(f)

def get_nbd(lng, lat):
    point = Point(lng, lat)    
    for feature in js['features']:
        polygon = shape(feature['geometry'])
        if polygon.contains(point):
            return feature['properties']['name']

gecode_firefox.py

"""
Description: Code Snippets for NYC Traffic Data Extraction
Author: Pritthijit Nath (nathzi1505)
GitHub Gist: https://gist.github.com/nathzi1505/c56abcb030e35059ee39c5ea66ba16d9

NOTE: RESULTS WILL BE OFF BY LITTLE
"""
    


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import pandas as pd
import time
import os

import re

def geocode(name, address):
    
    try:
        driver = webdriver.Firefox()

        driver.get("https://www.google.com/maps/")
        searchbox = driver.find_element_by_css_selector("#searchboxinput")

        searchbox.clear()
        searchbox.send_keys(address)
        searchbox.send_keys(Keys.ENTER)

        time.sleep(15)
        url = driver.current_url
        coordinates = re.findall(r'(@[\-0-9\.]*,[\-0-9\.]*)', url)[0].replace('@', '').split(',')
        coordinates = [np.float64(item) for item in coordinates]  

        driver.close()

        locations.append({
            "road": name,
            "lat": coordinates[0],
            "lng": coordinates[1]
        })    
    except:
        driver.close()
        geocode(name, address)

install_geckodriver.sh

cd /usr/bin
wget https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux64.tar.gz
tar -xvzf geckodriver-v0.26.0-linux64.tar.gz
rm geckodriver-v0.26.0-linux64.tar.gz
chmod +x geckodriver
export PATH=$PATH:/usr/bin/geckodriver

parallelize.py

from joblib import Parallel, delayed

locations = []
job = Parallel(n_jobs=-1, prefer="threads")(delayed(geocode)(road, road + ", New York City, USA") for road in roads)