#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import (division, print_function, absolute_import,
 unicode_literals)
__all__ = ["fetch"]
import gevent
from gevent import monkey
monkey.patch_all()
import os
import shutil
import requests
from itertools import product
from tempfile import NamedTemporaryFile
data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
filename = os.path.join(data_dir, "{year}-{month:02d}-{day:02d}-{n}.json.gz")
try:
 os.makedirs(data_dir)
except os.error:
 pass
url = "http://data.githubarchive.org/{year}-{month:02d}-{day:02d}-{n}.json.gz"
def fetch(year, month, day, n):
 kwargs = {"year": year, "month": month, "day": day, "n": n}
 local_fn = filename.format(**kwargs)
 # Skip if the file exists.
 if os.path.exists(local_fn):
 return
 # Download the remote file.
 remote = url.format(**kwargs)
 r = requests.get(remote)
 if r.status_code == requests.codes.ok:
 # Atomically write to disk.
 # http://stackoverflow.com/questions/2333872/ \
 # atomic-writing-to-file-with-python
 f = NamedTemporaryFile("wb", delete=False)
 f.write(r.content)
 f.flush()
 os.fsync(f.fileno())
 f.close()
 shutil.move(f.name, local_fn)
if __name__ == "__main__":
 for year, month in product(range(2011, 2014), range(1, 13)):
 jobs = [gevent.spawn(fetch, year, month, day, n)
 for n, day in product(range(1, 32), range(24))]
 gevent.joinall(jobs)
 print("Finished {0}-{1}".format(year, month))