fetch_data.py

import csv
import urllib.request
import codecs
import re
import sys
import math
import time
import json
import requests
import pyrebase
from time import gmtime, strftime
from datetime import datetime
from bs4 import BeautifulSoup

# Firebase config
config = {
  "apiKey": "APIKEY",
  "authDomain": "AUTHDOMAIN",
  "databaseURL": "URL",
  "storageBucket": "STORAGEBUCKET"
}
firebase = pyrebase.initialize_app(config)


# Get a reference to the auth service
email = ""
password = ""
auth = firebase.auth()
user = auth.sign_in_with_email_and_password(email, password)
user = auth.refresh(user['refreshToken'])
# target url
url = 'https://docs.google.com/spreadsheets/d/1pTgvUcZoaVcrdiFRgow4PdAURbUaCGnszL853UntAew/export?format=csv&id=1pTgvUcZoaVcrdiFRgow4PdAURbUaCGnszL853UntAew&gid=138064684'
# title fieldnames
title = []

mappingTable = {
    'Timestamp': 'Timestamp',
    '你之前去哪實習呢？': 'Name',
    # ...
}
while True:
    ### make url request
    ftpstream = urllib.request.urlopen(url)
    csvfile = csv.reader(codecs.iterdecode(ftpstream, 'utf-8'))
    line_counter = 1
    csv_rows = []
    csv_object = {}
    for row in csvfile:
        line_counter += 1
        if(line_counter == 6):
            title = row
            print(title)
        if(line_counter > 6):
            csv_object[line_counter - 7] = {mappingTable[title[i]]: row[i] for i in range(len(title))}
            csv_object[line_counter - 7]['ID'] = line_counter - 6

    ### Get a reference to the database service
    db = firebase.database()
    results = db.child('list').update(csv_object, user['idToken'])
    print('Fetch data, ', datetime.now())
    time.sleep(300)

### shut down the script
sys.stdout.flush()
typhoon_JSON.close()
sys.exit()