gitlab-extract.py 3.03 KB
Newer Older
1
#!/usr/bin/env python3
2
import argparse, pprint, sys
3
import requests
4
import parse_log
5

6
def last(it):
7
    r = None
8
9
10
11
    for i in it:
        r = i
    return r

12
13
14
def first(it):
    for i in it:
        return i
15
    return None
16
17
18

def req(path):
    url = '%s/api/v3/%s' % (args.server, path)
19
    return requests.get(url, headers={'PRIVATE-TOKEN': args.private_token})
20
21

# read command-line arguments
22
parser = argparse.ArgumentParser(description='Extract iris-coq build logs from GitLab')
23
24
parser.add_argument("-t", "--private-token",
                    dest="private_token", required=True,
25
                    help="The private token used to authenticate access.")
26
27
parser.add_argument("-s", "--server",
                    dest="server", default="https://gitlab.mpi-sws.org/",
28
                    help="The GitLab server to contact.")
29
parser.add_argument("-p", "--project",
30
                    dest="project", default="FP/iris-coq",
31
32
33
34
35
36
37
                    help="The name of the project on GitLab.")
parser.add_argument("-f", "--file",
                    dest="file", required=True,
                    help="Filename to store the load in.")
parser.add_argument("-c", "--commits",
                    dest="commits",
                    help="The commits to fetch. Default is everything since the most recent entry in the log file.")
38
args = parser.parse_args()
39
log_file = sys.stdout if args.file == "-" else open(args.file, "a")
40

41
42
43
44
# determine commit, if missing
if args.commits is None:
    if args.file == "-":
        raise Exception("If you do not give explicit commits, you have to give a logfile so that we can determine the missing commits.")
45
    last_result = last(parse_log.parse(open(args.file, "r"), parse_times = False))
46
47
    args.commits = "{}..origin/master".format(last_result.commit)

48
projects = req("projects")
49
project = first(filter(lambda p: p['path_with_namespace'] == args.project, projects.json()))
50
51
52
if project is None:
    sys.stderr.write("Project not found.\n")
    sys.exit(1)
53

54
for commit in parse_log.parse_git_commits(args.commits):
55
    print("Fetching {}...".format(commit))
56
57
58
    commit_data = req("/projects/{}/repository/commits/{}".format(project['id'], commit))
    if commit_data.status_code != 200:
        raise Exception("Commit not found?")
59
60
61
62
    builds = req("/projects/{}/repository/commits/{}/builds".format(project['id'], commit))
    if builds.status_code != 200:
        # no build
        continue
63
    build = first(sorted(builds.json(), key = lambda b: -int(b['id'])))
64
65
    if build is None or build['status'] == 'failed':
        # build failed (or missing...??)
66
        continue
67
68
69
    if build['status'] == 'running':
        # build still running, don't fetch this or any later commit
        break
70
    # now fetch the build times
71
    build_times = requests.get("{}/builds/{}/artifacts/file/build-time.txt".format(project['web_url'], build['id']))
72
    if build_times.status_code != 200:
73
        raise Exception("No artifact at build?")
74
75
76
    # Output in the log file format
    log_file.write("# {}\n".format(commit))
    log_file.write(build_times.text)
77
    log_file.flush()