dbUtils.py

# pip install sqlalchemy
# pip install pymysql
from sqlalchemy import create_engine
import pandas as pd

def get_mysql_conn(stream_results=True):
    user, password, host, port, db = 'user', 'pwd', 'host', "port", 'database'
    mysql_engine_str = 'mysql+pymysql://%s:%s@%s:%s/%s' % (user, password, host, port, db)
    engine = create_engine(mysql_engine_str, execution_options=dict(stream_results=True))
    conn = engine.connect()
    return conn

def query_mysql_to_dataframe(sql_query, stream_results=True, chunksize=1000):
    ''' use stream_results=True to avoid memory problem for server side if fetching large data
    Usage: query_mysql_to_dataframe('select * from table_1 limit 100000')
    '''
    dfs = []
    with get_mysql_conn(stream_results=stream_results) as conn:
        for table in pd.read_sql_query(sql_query, conn, chunksize=chunksize):
            dfs.append(table)
    df = pd.concat(dfs, ignore_index=True)
    return df