|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +''' |
| 4 | +export-contest -- Convenience script to export a contest (including metadata, |
| 5 | +teams and problems) from the command line. Defaults to using the CLI interface; |
| 6 | +Specify a DOMjudge API URL as to use that. |
| 7 | + |
| 8 | +Reads credentials from ~/.netrc when using the API. |
| 9 | + |
| 10 | +Part of the DOMjudge Programming Contest Jury System and licensed |
| 11 | +under the GNU GPL. See README and COPYING for details. |
| 12 | +''' |
| 13 | + |
| 14 | +import datetime |
| 15 | +import json |
| 16 | +import os |
| 17 | +import sys |
| 18 | +import time |
| 19 | +from argparse import ArgumentParser |
| 20 | +from concurrent.futures import ThreadPoolExecutor, as_completed |
| 21 | + |
| 22 | +sys.path.append('@domserver_libdir@') |
| 23 | +import dj_utils |
| 24 | + |
| 25 | +mime_to_extension = { |
| 26 | + 'application/pdf': 'pdf', |
| 27 | + 'application/zip': 'zip', |
| 28 | + 'image/jpeg': 'jpg', |
| 29 | + 'image/png': 'png', |
| 30 | + 'image/svg+xml': 'svg', |
| 31 | + 'text/plain': 'txt', |
| 32 | + 'video/mp4': 'mp4', |
| 33 | + 'video/mpeg': 'mpg', |
| 34 | + 'video/webm': 'webm', |
| 35 | +} |
| 36 | + |
| 37 | +def get_default_contest(): |
| 38 | + c_default = None |
| 39 | + |
| 40 | + contests = dj_utils.do_api_request('contests') |
| 41 | + if len(contests)>0: |
| 42 | + now = int(time.time()) |
| 43 | + for c in contests: |
| 44 | + if 'start_time' not in c or c['start_time'] is None: |
| 45 | + # Assume that a contest with start time unset will start soon. |
| 46 | + c['start_epoch'] = now + 1 |
| 47 | + else: |
| 48 | + c['start_epoch'] = datetime.datetime.fromisoformat(c['start_time']).timestamp() |
| 49 | + |
| 50 | + c_default = contests[0] |
| 51 | + for c in contests: |
| 52 | + if c_default['start_epoch']<=now: |
| 53 | + if c['start_epoch']<=now and c['start_epoch']>c_default['start_epoch']: |
| 54 | + c_default = c |
| 55 | + else: |
| 56 | + if c['start_epoch']<c_default['start_epoch']: |
| 57 | + c_default = c |
| 58 | + |
| 59 | + return c_default |
| 60 | + |
| 61 | + |
| 62 | +def download_file(file: dict, dir: str, default_name: str): |
| 63 | + print(f"Downloading '{file['href']}'") |
| 64 | + os.makedirs(dir, exist_ok=True) |
| 65 | + filename = file['filename'] if 'filename' in file else default_name |
| 66 | + dj_utils.do_api_request(file['href'], decode=False, output_file=f'{dir}/{filename}') |
| 67 | + |
| 68 | + |
| 69 | +def is_file(data) -> bool: |
| 70 | + ''' |
| 71 | + Check whether API data represents a FILE object. This is heuristic because |
| 72 | + no property is strictly required, but we need at least `href` to download |
| 73 | + the file, so if also we find one other property, we announce a winner. |
| 74 | + ''' |
| 75 | + if not isinstance(data, dict): |
| 76 | + return false |
| 77 | + return 'href' in data and ('mime' in data or 'filename' in data or 'hash' in data) |
| 78 | + |
| 79 | + |
| 80 | +files_to_download = [] |
| 81 | + |
| 82 | +def recurse_find_files(data, store_path: str, default_name: str): |
| 83 | + if isinstance(data, list): |
| 84 | + # Special case single element list for simpler default_name |
| 85 | + if len(data) == 1: |
| 86 | + recurse_find_files(data[0], store_path, default_name) |
| 87 | + else: |
| 88 | + for i, item in enumerate(data): |
| 89 | + recurse_find_files(item, store_path, f"{default_name}.{i}") |
| 90 | + elif isinstance(data, dict): |
| 91 | + if is_file(data): |
| 92 | + if 'mime' in data and data['mime'] in mime_to_extension: |
| 93 | + default_name += '.' + mime_to_extension[data['mime']] |
| 94 | + files_to_download.append((data, store_path, default_name)) |
| 95 | + else: |
| 96 | + for key, item in data.items(): |
| 97 | + recurse_find_files(item, store_path, f"{default_name}.{key}") |
| 98 | + |
| 99 | + |
| 100 | +def download_endpoint(name: str, path: str): |
| 101 | + ext = '.ndjson' if name == 'event-feed' else '.json' |
| 102 | + filename = name + ext |
| 103 | + |
| 104 | + print(f"Fetching '{path}' to '{filename}'") |
| 105 | + data = dj_utils.do_api_request(path, decode=False) |
| 106 | + with open(filename, 'wb') as f: |
| 107 | + f.write(data) |
| 108 | + |
| 109 | + if ext == '.json': |
| 110 | + data = json.loads(data) |
| 111 | + store_path = name |
| 112 | + if isinstance(data, list): |
| 113 | + for elem in data: |
| 114 | + recurse_find_files(elem, f"{store_path}/{elem['id']}", '') |
| 115 | + else: |
| 116 | + recurse_find_files(data, store_path, '') |
| 117 | + |
| 118 | + |
| 119 | +cid = None |
| 120 | +dir = None |
| 121 | + |
| 122 | +parser = ArgumentParser(description='Export a contest archive from DOMjudge via the API.') |
| 123 | +parser.add_argument('-c', '--cid', help="contest ID to export, defaults to last started, or else first non-started active contest") |
| 124 | +parser.add_argument('-d', '--dir', help="directory to write the contest archive to, defaults to contest ID in current directory") |
| 125 | +parser.add_argument('-u', '--url', help="DOMjudge API URL to use, if not specified use the CLI interface") |
| 126 | +args = parser.parse_args() |
| 127 | + |
| 128 | +if args.cid: |
| 129 | + cid = args.cid |
| 130 | +else: |
| 131 | + c = get_default_contest() |
| 132 | + if c is None: |
| 133 | + print("No contest specified nor an active contest found.") |
| 134 | + exit(1) |
| 135 | + else: |
| 136 | + cid = c['id'] |
| 137 | + |
| 138 | +if args.dir: |
| 139 | + dir = args.dir |
| 140 | +else: |
| 141 | + dir = cid |
| 142 | + |
| 143 | +if args.url: |
| 144 | + dj_utils.domjudge_api_url = args.url |
| 145 | + |
| 146 | +user_data = dj_utils.do_api_request('user') |
| 147 | +if 'admin' not in user_data['roles']: |
| 148 | + print('Your user does not have the \'admin\' role, can not export.') |
| 149 | + exit(1) |
| 150 | + |
| 151 | +if os.path.exists(dir): |
| 152 | + print(f'Export directory \'{dir}\' already exists, will not overwrite.') |
| 153 | + exit(1) |
| 154 | + |
| 155 | +os.makedirs(dir) |
| 156 | +os.chdir(dir) |
| 157 | + |
| 158 | +contest_path = f'contests/{cid}' |
| 159 | + |
| 160 | +# Custom endpoints: |
| 161 | +download_endpoint('api', '') |
| 162 | +download_endpoint('contest', contest_path) |
| 163 | +download_endpoint('event-feed', f'{contest_path}/event-feed?stream=false') |
| 164 | + |
| 165 | +for endpoint in [ |
| 166 | + 'access', |
| 167 | + 'accounts', |
| 168 | + 'awards', |
| 169 | +# 'balloons', This is a DOMjudge specific endpoint |
| 170 | + 'clarifications', |
| 171 | +# 'commentary', Not implemented in DOMjudge |
| 172 | + 'groups', |
| 173 | + 'judgement-types', |
| 174 | + 'judgements', |
| 175 | + 'languages', |
| 176 | + 'organizations', |
| 177 | +# 'persons', Not implemented in DOMjudge |
| 178 | + 'problems', |
| 179 | + 'runs', |
| 180 | + 'scoreboard', |
| 181 | + 'state', |
| 182 | + 'submissions', |
| 183 | + 'teams', |
| 184 | + ]: |
| 185 | + download_endpoint(endpoint, f"{contest_path}/{endpoint}") |
| 186 | + |
| 187 | +with ThreadPoolExecutor(20) as executor: |
| 188 | + futures = [executor.submit(download_file, *item) for item in files_to_download] |
| 189 | + for future in as_completed(futures): |
| 190 | + future.result() # So it can throw any exception |
0 commit comments