Examples for Mistral observational data ingestion flows

mistral_dbadb_insert.py

# Legge messaggi in BUFR o JSON da stdin e li salva in un dbadb
# Utilizzo: mistral_dbadb_insert.py [BUFR|JSON] DBADSN
#
# Author: Emanuele Di Giacomo <[email protected]>
# License: GPLv2+

import argparse
import sys

import dballe

if __name__ == '__main__':
    parser = argparser.ArgumentParser()
    parser.add_argument("format", choices=["BUFR", "JSON"])
    parser.add_argument("dbadb", type=dballe.DB.connect)
    args = parser.parse_args()
    
    importer = dballe.Importer(args.format)

    with importer.from_file(sys.stdin.buffer) as fp:
        for msgs in fp:
            for msg in msgs:
                with args.dbadb.transaction() as tr:
                    tr.import_messages(msg, import_attributes=True, update_station=True, overwrite=True)

mistral_filter_report.py

# Legge messaggi in BUFR o JSON da stdin e li scrive su:
# - stdout se appartiene alla rete
# - stderr se non appartiene alla rete
#
# Utilizzo: mistral_filter_report.py [BUFR|JSON] REPORT_NAME
#
# Author: Emanuele Di Giacomo <[email protected]>
# License: GPLv2+

import argparse
import sys

import dballe

if __name__ == '__main__':
    parser = argparser.ArgumentParser()
    parser.add_argument("format", choices=["BUFR", "JSON"])
    parser.add_argument("report")
    args = parser.parse_args()
    
    importer = dballe.Importer(args.format)
    exporter = dballe.Exporter("BUFR")

    with importer.from_file(sys.stdin.buffer) as fp:
        for msgs in fp:
            for msg in msgs:
                binarydata = exporter.to_binary(msg)
                if msg.report == args.report:
                    sys.stdout.buffer.write(binarydata)
                else:
                    sys.stderr.buffer.write(binarydata)