103 lines
2.8 KiB
Python
103 lines
2.8 KiB
Python
|
import json
|
||
|
|
import mariadb
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
from urllib.request import urlopen
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
from datetime import datetime
|
||
|
|
|
||
|
|
|
||
|
|
def read_config(file="config.json"):
|
||
|
|
"""
|
||
|
|
Read config file to get database configuration and other stuff.
|
||
|
|
"""
|
||
|
|
with open(file, "r") as f:
|
||
|
|
config = json.load(f)
|
||
|
|
return config
|
||
|
|
|
||
|
|
|
||
|
|
def prepare_db():
|
||
|
|
"""
|
||
|
|
Setup database table, if it not exists.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
conn = mariadb.connect(
|
||
|
|
user=config["db_user"],
|
||
|
|
password=config["db_passwd"],
|
||
|
|
host=config["db_host"],
|
||
|
|
port=config["db_port"],
|
||
|
|
database=config["db_db"]
|
||
|
|
)
|
||
|
|
except mariadb.Error as e:
|
||
|
|
print(f"Error connecting to MariaDB Platform: {e}")
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
cur = conn.cursor()
|
||
|
|
cur.execute("SHOW TABLES")
|
||
|
|
|
||
|
|
table_exists = False
|
||
|
|
for table in cur:
|
||
|
|
if table == config["db_table"]:
|
||
|
|
table_exists = True
|
||
|
|
|
||
|
|
if not table_exists:
|
||
|
|
cur.execute("CREATE TABLE ? (timestamp TIMESTAMP PRIMARY KEY, level_cm INT, flow_m3_s INT)", (config["db_table"]))
|
||
|
|
|
||
|
|
conn.close()
|
||
|
|
|
||
|
|
|
||
|
|
def write_db(timestamp, level: int, flow: int):
|
||
|
|
"""
|
||
|
|
Write data set to database.
|
||
|
|
@param timestamp: timestamp of data set
|
||
|
|
@param level: water level in cm
|
||
|
|
@param flow: water flow in m3/s
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
conn = mariadb.connect(
|
||
|
|
user=config["db_user"],
|
||
|
|
password=config["db_passwd"],
|
||
|
|
host=config["db_host"],
|
||
|
|
port=config["db_port"],
|
||
|
|
database=config["db_db"]
|
||
|
|
)
|
||
|
|
except mariadb.Error as e:
|
||
|
|
print(f"Error connecting to MariaDB Platform: {e}")
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
cur = conn.cursor()
|
||
|
|
try:
|
||
|
|
cur.execute("INSERT INTO ? (timestamp,level_cm,flow_m3_s) VALUES (?, ?, ?)", (config["db_table"], timestamp, level, flow))
|
||
|
|
except mariadb.Error as e:
|
||
|
|
print(f"Error: {e}")
|
||
|
|
finally:
|
||
|
|
conn.close()
|
||
|
|
|
||
|
|
|
||
|
|
def retrieve(url: str):
|
||
|
|
"""
|
||
|
|
Get webpage and parse it to get timestamp, water level and flow.
|
||
|
|
@param url: the URL to call
|
||
|
|
@return: 3-tuple with timestamp, level, flow
|
||
|
|
"""
|
||
|
|
page = urlopen(url)
|
||
|
|
html = page.read().decode("utf-8")
|
||
|
|
soup = BeautifulSoup(html, "html.parser")
|
||
|
|
|
||
|
|
latest = soup.select(".quickbarTable tbody")[0].select("tr")[0]
|
||
|
|
latest_ts = latest.select("td")[0].text
|
||
|
|
ts = datetime.strptime(latest_ts.strip(), "%d.%m.%Y %H:%M")
|
||
|
|
latest_level = latest.select("td")[1].text
|
||
|
|
latest_flow = latest.select("td")[2].text
|
||
|
|
|
||
|
|
return ts, int(latest_level), int(latest_flow)
|
||
|
|
|
||
|
|
|
||
|
|
config = read_config()
|
||
|
|
# prepare_db()
|
||
|
|
while True:
|
||
|
|
ts, latest_level, latest_flow = retrieve(config["url"])
|
||
|
|
# write_db(ts, latest_level, latest_flow)
|
||
|
|
print(str(ts) + ": " + str(latest_level) + "cm " + str(latest_flow) + "m3/s")
|
||
|
|
time.sleep(config["sleep_sec"])
|