Created
June 23, 2017 20:06
-
-
Save deknos/e4563da7ef22004539ed10736fa6233a to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python -Es | |
import sys | |
import os | |
import os.path | |
import json | |
import json_delta | |
import urllib2 | |
import time | |
lastversionfile='last_version' | |
downloadurlprefix='https://multimobil-core.mvg.de/service/v5/networkState/networkState?MVG_RAD=' | |
startnumber=1 | |
curnumber=-1 | |
wartezeit=1 | |
if os.path.exists(lastversionfile): | |
# Ueberpruefe ob es schon einen stamp gibt, wo etwas heruntergeladen wurde | |
fh=open(lastversionfile) | |
line=fh.readline().strip('\n') # only one number | |
if line.isdigit(): | |
curnumber=line | |
fh.close() | |
def getnewjsondata(url): | |
# laedt daten einer url und erzeugt ein json objekt | |
try: | |
response=urllib2.urlopen(url) | |
ujd = response.read() | |
return json.loads(ujd) | |
except : | |
print "HTTP connection had a failure" | |
return None | |
# Hole entweder die daten seit dem letzten stamp oder seit dem anfang | |
if curnumber != -1 : | |
importedjson = getnewjsondata(downloadurlprefix+curnumber) | |
else: | |
importedjson = getnewjsondata(downloadurlprefix+str(startnumber)) | |
curnumber=startnumber | |
if importedjson == None: | |
print "First fetch failed, stop" | |
sys.exit(1) | |
def getnumberfromjsonstructure(ij): | |
# laedt stamp aus jsonobjekt | |
return ij['networkStateVersions']['MVG_RAD'] | |
def writedata(number,jsonstruct): | |
fh=open(str(number), 'w') | |
fh.writelines(json.dumps(jsonstruct)) | |
fh.close() | |
def savedataifdifferent(versionfile, newdata, curnum): | |
# hole stamp | |
newnum = getnumberfromjsonstructure(newdata) | |
# speichere falls anders | |
if newnum != curnum : | |
writedata(newnum,newdata) | |
writedata(versionfile,newnum) | |
return True, newnum | |
else: | |
return False, newnum | |
# sichert alte nummer und json objekt | |
lastnumber=curnumber | |
jsonprev=importedjson | |
different, curnumber = savedataifdifferent(lastversionfile,importedjson,curnumber) | |
# Prolog ist fertig, initiales json objekt ist fertig | |
# jetzt kann versucht werden, weitere daten zu laden und ein delta zu erzeugen | |
while 1: | |
time.sleep(wartezeit) | |
importedjson = getnewjsondata(downloadurlprefix+str(curnumber)) | |
if importedjson == None: | |
continue | |
different, newnum = savedataifdifferent(lastversionfile,importedjson, curnumber) | |
if different == True: | |
diff = json_delta.diff(jsonprev,importedjson) | |
writedata(str(curnumber)+"-"+str(newnum),diff) | |
jsonprev=importedjson | |
curnumber=newnum | |
print "New Stamp:"+str(curnumber)+" generated also a diff" | |
else: | |
print "No new data was generated" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment