ÖHB DB
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
oehbdb/main.py

559 lines
24 KiB

import argparse
import datetime
import json
import logging
import os
import sys
from copy import copy
from email.mime.text import MIMEText
from subprocess import Popen, PIPE
from oehbdb import store, data
logger = logging.getLogger()
def dir_path(string):
if os.path.isdir(string):
return string
else:
raise NotADirectoryError(string)
def setup_logging(verbosity):
base_loglevel = 30
verbosity = min(verbosity, 2)
loglevel = base_loglevel - (verbosity * 10)
logging.basicConfig(level=loglevel,
format='%(message)s')
def setup_file_logging(log_file, log_level):
logger.info(f"Writing logs to {log_file}")
f_handler = logging.FileHandler(log_file)
f_handler.setLevel(log_level)
f_format = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')
f_handler.setFormatter(f_format)
logging.getLogger('').addHandler(f_handler)
def send_notification(notices, email: str):
msg = MIMEText(f"""
OEHBDB Notification
{notices}
""")
msg["To"] = email
msg["Subject"] = "OEHBDB Notification"
p = Popen(["/usr/sbin/sendmail", "-t"], stdin=PIPE)
p.communicate(msg.as_bytes())
def cmd_import(args: argparse.Namespace):
from oehbdb import scrapliga
if args.log_file:
setup_file_logging(args.log_file, logging.INFO)
feeds = scrapliga.read_feeds(args.feed_dir.rstrip('/'))
store.open(args.db_file)
championships_added = []
championships_updated = []
championships_deleted = []
unprocessed_championships = copy(data.objects(data.Championship)) # type: set[data.Championship]
for item in feeds['championships']:
obj = store.get_championship_by_nuliga_id(item['nuliga_id'])
if not obj:
obj = scrapliga.create_championship_object(item)
store.add_data(obj)
championships_added.append(obj.name)
else:
if scrapliga.update_object(obj, item):
championships_updated.append(obj.name)
unprocessed_championships.remove(obj)
for obj in unprocessed_championships:
store.del_data(obj)
championships_deleted.append(obj.name)
del obj
clubs_added = []
clubs_updated = []
clubs_deleted = []
unprocessed_clubs = copy(data.objects(data.Club)) # type: set[data.Club]
for club_item in feeds['clubs']:
obj = store.get_club_by_nuliga_id(club_item['nuliga_id'])
if not obj:
obj = scrapliga.create_club_object(club_item)
store.add_data(obj)
clubs_added.append(obj.name)
else:
if scrapliga.update_object(obj, club_item):
clubs_updated.append(obj.name)
unprocessed_clubs.remove(obj)
for obj in unprocessed_clubs:
store.del_data(obj)
clubs_deleted.append(obj.name)
del obj
groups_added = []
groups_updated = []
groups_deleted = []
unprocessed_groups = copy(data.objects(data.Group)) # type: set[data.Group]
for group_item in feeds['groups']:
obj = store.get_group_by_nuliga_id(group_item['nuliga_id'])
if not obj:
obj = scrapliga.create_group_object(group_item)
store.add_data(obj)
groups_added.append("{} - {}".format(obj.championship.short_name,obj.name))
else:
if scrapliga.update_object(obj, group_item):
groups_updated.append("{} - {}".format(obj.championship.short_name, obj.name))
if obj in unprocessed_groups:
unprocessed_groups.remove(obj)
else:
logger.warning(f"possible inconsistency: group {obj.name} was found in store "
f"(nuliga id: {obj.nuliga_id}, but not in data.objects (id {obj.id})")
for obj in unprocessed_groups:
store.del_data(obj)
groups_deleted.append("{} - {}".format(obj.championship.short_name, obj.name))
del obj
courts_added = []
courts_updated = []
courts_deleted = []
unprocessed_courts = copy(data.objects(data.Court))
for court_item in feeds['courts']:
obj = store.get_court_by_nuliga_id(court_item['nuliga_id'])
if not obj:
obj = scrapliga.create_court_object(court_item)
store.add_data(obj)
courts_added.append(obj.name)
else:
if scrapliga.update_object(obj, court_item):
courts_updated.append(obj.name)
unprocessed_courts.remove(obj)
for obj in unprocessed_courts:
store.del_data(obj)
courts_deleted.append(obj.name)
del obj
teams_added = []
teams_updated = []
teams_deleted = []
unprocessed_teams = copy(data.objects(data.Team))
for team_item in feeds['teams']:
obj = store.get_team_by_nuliga_id(team_item['nuliga_id'])
if not obj:
obj = scrapliga.create_team_object(team_item)
store.add_data(obj)
teams_added.append("{} - {}".format(obj.club.name, obj.name))
else:
if scrapliga.update_object(obj, team_item):
teams_updated.append("{} - {}".format(obj.club.name, obj.name))
unprocessed_teams.remove(obj)
for obj in unprocessed_teams:
store.del_data(obj)
teams_deleted.append("{} - {}".format(obj.club.name, obj.name))
del obj
matches_added = 0
matches_updated = 0
matches_deleted = 0
unprocessed_matches = copy(data.objects(data.Match))
for match_item in feeds['matches']:
if match_item['championship'].startswith('Steirischer Handballverband'):
# Data from StHV is botched up. Causes more problems than useful information
continue
if 'nuliga_id' in match_item and match_item['nuliga_id']:
match = store.get_match_by_nuliga_id(match_item['nuliga_id'])
else:
if not 'match_number' in match_item:
logger.warning("Match not imported: no match number found %s" % json.dumps(match_item))
continue
elif "spielfrei" in [match_item['team_a'], match_item['team_b']]:
logger.debug("Match not imported: spielfrei %s - %s: %s vs. %s" % (
match_item['championship'], match_item['group'], match_item['team_a'], match_item['team_b']))
continue
elif match_item['match_number'] == "0":
logger.warning("Match not imported: match number 0 %s - %s: %s vs. %s" % (
match_item['championship'], match_item['group'], match_item['team_a'], match_item['team_b']))
continue
match = store.get_match_by_nuliga_item(match_item)
if not match:
match = scrapliga.create_match_object(match_item)
store.add_data(match)
matches_added += 1
else:
if scrapliga.update_object(match, match_item):
matches_updated += 1
unprocessed_matches.remove(match)
for obj in unprocessed_matches:
store.del_data(obj)
del obj
matches_deleted += 1
logger.info("Import stats [added/updated/deleted]")
logger.info("Championships: {}/{}/{}".format(
len(championships_added), len(championships_updated), len(championships_deleted)
))
logger.info("Groups : {}/{}/{}".format(
len(groups_added), len(groups_updated), len(groups_deleted)
))
logger.info("Clubs : {}/{}/{}".format(
len(clubs_added), len(clubs_updated), len(clubs_deleted)
))
logger.info("Courts : {}/{}/{}".format(
len(courts_added), len(courts_updated), len(courts_deleted)
))
logger.info("Teams : {}/{}/{}".format(
len(teams_added), len(teams_updated), len(teams_deleted)
))
logger.info("Matches : %d/%d/%d" % (matches_added, matches_updated, matches_deleted))
if args.notify_mail:
notifications = []
if len(championships_added) > 0:
notifications.append("Championships added:\n{}".format("\n".join(championships_added)))
if len(championships_updated) > 0:
notifications.append("Championships updated:\n{}".format("\n".join(championships_updated)))
if len(championships_deleted) > 0:
notifications.append("Championships deleted:\n{}".format("\n".join(championships_deleted)))
if len(groups_added) > 0:
notifications.append("Groups added:\n{}".format("\n".join(groups_added)))
if len(groups_updated) > 0:
notifications.append("Groups updated:\n{}".format("\n".join(groups_updated)))
if len(groups_deleted) > 0:
notifications.append("Groups deleted:\n{}".format("\n".join(groups_deleted)))
if len(clubs_added) > 0:
notifications.append("Clubs added:\n{}".format("\n".join(clubs_added)))
if len(clubs_updated) > 0:
notifications.append("Clubs updated:\n{}".format("\n".join(clubs_updated)))
if len(clubs_deleted) > 0:
notifications.append("Clubs deleted:\n{}".format("\n".join(clubs_deleted)))
if len(courts_added) > 0:
notifications.append("Courts added:\n{}".format("\n".join(courts_added)))
if len(courts_updated) > 0:
notifications.append("Courts updated:\n{}".format("\n".join(courts_updated)))
if len(courts_deleted) > 0:
notifications.append("Courts deleted:\n{}".format("\n".join(courts_deleted)))
if len(teams_added) > 0:
notifications.append("Teams added:\n{}".format("\n".join(teams_added)))
if len(teams_updated) > 0:
notifications.append("Teams updated:\n{}".format("\n".join(teams_updated)))
if len(teams_deleted) > 0:
notifications.append("Teams deleted:\n{}".format("\n".join(teams_deleted)))
if notifications:
send_notification("\n\n".join(notifications), args.notify_mail)
sys.exit(0)
def cmd_show(args: argparse.Namespace):
store.open(args.db_file)
obj = store.get_obj_by_id(args.obj_id)
if not obj:
print("Object not found", file=sys.stderr)
sys.exit(2)
print("""
%s
""" % (obj.output()))
sys.exit(0)
def cmd_delete(args: argparse.Namespace):
store.open(args.db_file)
data.object_by_id()
obj = store.get_obj_by_id(args.obj_id)
if not obj:
print("Object not found", file=sys.stderr)
sys.exit(2)
if store.del_data(obj):
logger.info("Object %s (%s) deleted" % (obj.name, obj.id))
sys.exit(0)
else:
logger.error("Failed to delete object %s (%s)" % (obj.name, obj.id))
sys.exit(2)
def cmd_list(args: argparse.Namespace):
if args.obj_type == 'championships':
store.open(args.db_file)
championships = data.objects(data.Championship) # type: set[data.Championship]
if not championships:
print("No championships found")
sys.exit(0)
championships = list(championships) # type: list[data.Championship]
championships.sort(key=lambda i: i.name)
print("Championships")
for championship in championships:
print("%s -- %s (%s)" % (championship.short_name, championship.name, championship.id))
sys.exit(0)
elif args.obj_type == 'groups':
store.open(args.db_file)
championship = data.object_by_id(data.Championship, args.championship)
if not championship:
raise argparse.ArgumentError(None, "Bad selector")
if not isinstance(championship, data.Championship):
raise argparse.ArgumentError(None, "Unsupported selector type %s" % championship.__class__.__name__)
groups = championship.groups()
if not groups:
print("No groups found for %s" % championship.name)
sys.exit(0)
print("Groups of %s" % championship.name)
groups = list(groups)
groups.sort(key=lambda i: i.name)
for group in groups:
print("%s (%s)" % (group.name, group.id))
elif args.obj_type == 'teams':
store.open(args.db_file)
if not args.selector:
teams = data.objects(data.Team) # type: set[data.Team]
print("All teams")
else:
group = data.object_by_id(data.Group, args.selector)
championship = data.object_by_id(data.Championship, args.selector)
if group:
teams = group.teams()
if not teams:
print("No team found for group %s - %s" % (group.championship.name, group.name))
sys.exit(0)
print("Teams of group %s - %s" % (group.championship.name, group.name))
elif championship:
teams = set()
for group in championship.groups():
teams.update(group.teams())
if not teams:
print("No team found for championship %s" % championship.name)
sys.exit(0)
print("Teams of championship %s" % championship.name)
else:
raise argparse.ArgumentError(None, "Bad selector")
team_list = list(teams)
team_list.sort(key=lambda i: i.match_display_name)
for team in team_list:
if not team.club:
print("[club missing] - %s (%s)" % (team.name, team.id))
elif team.match_display_name == team.club.name:
print("%s (%s) (%s)" % (team.match_display_name, team.name, team.id))
else:
print("%s (%s - %s) (%s)" % (team.match_display_name, team.club.name, team.name, team.id))
elif args.obj_type == 'clubs':
store.open(args.db_file)
if not args.selector:
clubs = data.objects(data.Club)
print("All clubs")
else:
group = data.object_by_id(data.Group, args.selector)
championship = data.object_by_id(data.Championship, args.selector)
if group:
clubs = group.clubs()
if not clubs:
print("No club found for group %s - %s" % (group.championship, group.name))
sys.exit(0)
print("Clubs of group %s - %s" % (group.championship, group.name))
elif championship:
clubs = set()
for group in championship.groups():
clubs.update(group.clubs())
if not clubs:
print("No club found for championship %s" % championship.name)
sys.exit(0)
print("Clubs of championship %s" % championship.name)
else:
raise argparse.ArgumentError(None, "Bad selector")
clubs = list(clubs)
clubs.sort(key=lambda i: i.name)
for club in clubs:
if not club:
print("INVALID CLUB")
else:
print("%s (%s)" % (club.name, club.id))
elif args.obj_type == 'matches':
store.open(args.db_file)
championship = data.object_by_id(data.Championship, args.championship) # type: data.Championship
if not championship:
raise argparse.ArgumentError(
None, "Championship not found %s" % args.championship)
if not args.selector:
matches = championship.matches()
print("All matches")
else:
group = data.object_by_id(data.Group, args.selector) # type: data.Group
club = data.object_by_id(data.Club, args.selector) # type: data.Club
if group:
if group.championship != championship:
raise argparse.ArgumentError(
None, "Group %s (%s) is not of championship %s" % (group.name, group.id, championship.name))
matches = group.matches()
if not matches:
print("No matches found for group %s - %s" % (championship.name, group.name))
sys.exit(0)
print("Matches of group %s - %s" % (championship.name, group.name))
elif club:
if club not in championship.clubs():
raise argparse.ArgumentError(
None, "Club %s (%s) is not listed in championship %s" % (club.name, club.id, championship.name))
matches = set()
for match in championship.matches():
if match.team_b().club == club or match.team_a().club == club:
matches.add(match)
if not matches:
print("No matches found for club %s in %s" % (club.name, championship.name))
sys.exit(0)
print("Matches of club %s in %s" % (club.name, championship.name))
else:
raise argparse.ArgumentError(None, "Bad selector")
match_list = list(matches)
mindate = datetime.datetime.fromtimestamp(0)
match_list.sort(key=lambda i: i.date or mindate)
for match in match_list:
if match.date:
print("%s: %s vs. %s (%s)" % (match.date, match.team_a().match_display_name, match.team_b().match_display_name, match.id))
else:
print("[kein Datum]: %s vs. %s (%s)" % (match.team_a().match_display_name, match.team_b().match_display_name, match.id))
def cmd_standings(args: argparse.Namespace):
store.open(args.db_file)
group = data.object_by_id(data.Group, args.group) # type: data.Group
if not group:
print("Group not found", file=sys.stderr)
sys.exit(2)
if not group.standings():
print("No standings found for group {0}".format(group.name), file=sys.stderr)
sys.exit(2)
print("{} - {}".format(group.championship.name, group.name))
print("Standings as of {0}\n".format(group.updated))
team_width = 0
for standing in group.standings():
team_width = max(team_width, len(standing.team.match_display_name))
header = "Rank {:<" + str(team_width) + "} P W D L G+ G- Diff Points"
row_comp_str = "{:>2} {:<" + str(team_width) + "} {:>2} {:>2} {:>2} {:>2} {:>3} {:>3} {:>4} {:>3}"
row_noncomp_str = "{:>2} {:<" + str(team_width) + "} -- non competitive --"
print(header.format('Team'))
for standing in group.standings():
if isinstance(standing, data.GroupStandingCompetitive):
print(row_comp_str.format(standing.rank, standing.team.match_display_name, standing.played,
standing.won, standing.drawn, standing.lost,
standing.goals_for, standing.goals_against, standing.goals_diff,
standing.points))
else:
print(row_noncomp_str.format(standing.rank, standing.team.match_display_name))
print("\nP: plays, W: won, D: drawn, L: lost\nG+: goals for, G-: goals against, Diff: goals difference")
sys.exit(0)
def cmd_backup(args: argparse.Namespace):
from oehbdb import backup
setup_file_logging(args.log_file, logging.INFO)
store.open(args.db_file)
backup.backup(args.backup_file)
def cmd_restore(args: argparse.Namespace):
from oehbdb import backup
if not os.path.exists(args.backup_file):
logger.error("File not found %s" % args.backup_file)
sys.exit(1)
store.open(args.db_file)
backup.restore(args.backup_file)
def cmd_test_mail(args: argparse.Namespace):
send_notification('Testmail', args.email)
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='OEHB DB')
parser.add_argument('-v', '--verbose',
action='count',
dest='verbosity',
default=0,
help="verbose output (repeat for increased verbosity)")
parser.add_argument('-q', '--quiet',
action='store_const',
const=-1,
default=0,
dest='verbosity',
help="quiet output (show errors only)")
parser.add_argument('--db', default="oehbdb.db", dest='db_file', help="database file to use")
sub_parsers = parser.add_subparsers(dest="command", help='Command', required=True)
parser_import = sub_parsers.add_parser('import', help='Import scrapliga feeds')
parser_import.add_argument('feed_dir', metavar='DIR', type=str, help='Directory holding scrapliga feeds')
parser_import.add_argument('--log-file', type=str, dest='log_file', help="log to file")
parser_import.add_argument('--notify', metavar="EMAIL", type=str, dest='notify_mail', default=None,
help="Send notification email on changes")
parser_show = sub_parsers.add_parser('show', help='Show database object')
show_obj_id_arg = parser_show.add_argument('obj_id', metavar='ID', type=str, help='Object ID')
list_parser = sub_parsers.add_parser('list', help='List database objects')
list_sub_parsers = list_parser.add_subparsers(dest="obj_type", help='Object type')
championship_parser = list_sub_parsers.add_parser('championships', help='List championships')
group_parser = list_sub_parsers.add_parser('groups', help='List groups')
group_parser.add_argument('championship', metavar='SELECTOR', type=str, default=None,
help='Championship ID to list groups of')
teams_parser = list_sub_parsers.add_parser('teams', help='List teams')
teams_parser.add_argument('selector', metavar='SELECTOR', nargs='?', default=None,
help='Group ID or Championship ID to list teams of. Without selector all teams get '
'listed')
list_clubs_parser = list_sub_parsers.add_parser('clubs', help='List clubs')
list_clubs_parser.add_argument('selector', metavar='SELECTOR', nargs='?', default=None,
help='Group ID or Championship ID to list clubs of. Without selector all clubs get '
'listed.')
list_matches_parser = list_sub_parsers.add_parser('matches', help='List matches')
list_matches_parser.add_argument('championship', metavar='CHAMPIONSHIP',
help='Championship ID to list matches of.')
list_matches_parser.add_argument('selector', metavar='SELECTOR', nargs='?', default=None,
help='Group ID or Team ID to list matches of. Without selector all championship '
'matches get listed.')
parser_standings = sub_parsers.add_parser('standings', help='Show group standings')
parser_standings.add_argument('group', metavar='GROUP', type=str, help='Group ID')
parser_del = sub_parsers.add_parser('delete', help='Delete database object')
del_obj_id_arg = parser_del.add_argument('obj_id', metavar='ID', type=str, help='Object ID')
parser_backup = sub_parsers.add_parser('backup', help='backup database')
parser_backup.add_argument('backup_file', metavar='FILE', type=str, help='Backup file')
parser_backup.add_argument('--log-file', type=str, dest='log_file', help="log to file")
parser_restore = sub_parsers.add_parser('restore', help='restore database from backup')
parser_restore.add_argument('backup_file', metavar='FILE', type=str, help='Backup file')
parser_restore = sub_parsers.add_parser('test-mail', help='Send test notification mail')
parser_restore.add_argument('email', metavar='EMAIL', type=str, help='Recipient address')
args = parser.parse_args()
setup_logging(args.verbosity)
if args.command == 'import':
cmd_import(args)
elif args.command == 'show':
if len(args.obj_id) < 8:
raise argparse.ArgumentError(show_obj_id_arg, 'too short')
if len(args.obj_id) > 8:
raise argparse.ArgumentError(show_obj_id_arg, 'too long')
cmd_show(args)
elif args.command == 'list':
cmd_list(args)
elif args.command == 'standings':
cmd_standings(args)
elif args.command == 'delete':
if len(args.obj_id) < 8:
raise argparse.ArgumentError(del_obj_id_arg, 'too short')
if len(args.obj_id) > 8:
raise argparse.ArgumentError(del_obj_id_arg, 'too long')
cmd_delete(args)
elif args.command == 'backup':
cmd_backup(args)
elif args.command == 'restore':
cmd_restore(args)
elif args.command == 'test-mail':
cmd_test_mail(args)