gitlab-extract.py 4.37 KB
Newer Older
1
#!/usr/bin/env python3
2
import argparse, pprint, sys, glob, zipfile
3
import requests
4
import parse_log
5

6
def last(it):
7
    r = None
8
9
10
11
    for i in it:
        r = i
    return r

12
13
14
def first(it):
    for i in it:
        return i
15
    return None
16
17
18

def req(path):
    url = '%s/api/v3/%s' % (args.server, path)
19
20
21
    r = requests.get(url, headers={'PRIVATE-TOKEN': args.private_token})
    r.raise_for_status()
    return r
22
23

# read command-line arguments
24
parser = argparse.ArgumentParser(description='Extract iris-coq build logs from GitLab')
25
26
parser.add_argument("-t", "--private-token",
                    dest="private_token", required=True,
27
                    help="The private token used to authenticate access.")
28
29
parser.add_argument("-s", "--server",
                    dest="server", default="https://gitlab.mpi-sws.org/",
30
                    help="The GitLab server to contact.")
31
parser.add_argument("-p", "--project",
32
                    dest="project", default="FP/iris-coq",
33
34
35
36
37
38
39
                    help="The name of the project on GitLab.")
parser.add_argument("-f", "--file",
                    dest="file", required=True,
                    help="Filename to store the load in.")
parser.add_argument("-c", "--commits",
                    dest="commits",
                    help="The commits to fetch. Default is everything since the most recent entry in the log file.")
40
41
42
parser.add_argument("-a", "--artifacts",
                    dest="artifacts",
                    help="Location of the artifacts (following GitLab's folder structure).  If not given (which should be the common case), the artifacts will be downloaded from GitLab.")
43
args = parser.parse_args()
44
log_file = sys.stdout if args.file == "-" else open(args.file, "a")
45

46
47
48
49
# determine commit, if missing
if args.commits is None:
    if args.file == "-":
        raise Exception("If you do not give explicit commits, you have to give a logfile so that we can determine the missing commits.")
50
    last_result = last(parse_log.parse(open(args.file, "r"), parse_times = parse_log.PARSE_NOT))
51
52
    args.commits = "{}..origin/master".format(last_result.commit)

Ralf Jung's avatar
Ralf Jung committed
53
projects = req("projects?per_page=512")
54
project = first(filter(lambda p: p['path_with_namespace'] == args.project, projects.json()))
55
56
57
if project is None:
    sys.stderr.write("Project not found.\n")
    sys.exit(1)
58

59
BREAK = False
60
for commit in parse_log.parse_git_commits(args.commits):
61
62
    if BREAK:
        break
63
    print("Fetching {}...".format(commit))
64
65
66
    commit_data = req("/projects/{}/repository/commits/{}".format(project['id'], commit))
    if commit_data.status_code != 200:
        raise Exception("Commit not found?")
67
68
    builds = req("/projects/{}/repository/commits/{}/builds".format(project['id'], commit))
    if builds.status_code != 200:
69
70
71
72
73
74
75
76
77
78
79
        raise Exception("Build not found?")
    # iterate over builds by decreasing ID, and look for the artifact
    for build in builds.json():
        if build['status'] in ('created', 'pending', 'running'):
            # build still not yet done, don't fetch this or any later commit
            BREAK = True
            break
        if build['status'] != 'success':
            # build failed or cancelled, skip to next
            continue
        # now fetch the build times
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
        if args.artifacts:
            artifact_zip = glob.glob('{}/*/{}/{}/artifacts.zip'.format(args.artifacts, project['id'], build['id']))
            if not artifact_zip:
                # no artifact at this build, try another one
                continue
            assert len(artifact_zip) == 1, "Found too many artifacts"
            artifact_zip = artifact_zip[0]
            with zipfile.ZipFile(artifact_zip) as artifact:
                with artifact.open('build-time.txt') as build_times:
                    # Output into log file
                    log_file.write("# {}\n".format(commit))
                    log_file.write(build_times.read().decode('UTF-8'))
                    log_file.flush()
        else:
            build_times = requests.get("{}/builds/{}/artifacts/raw/build-time.txt".format(project['web_url'], build['id']))
            if build_times.status_code != 200:
                # no artifact at this build, try another one
                continue
            # Output in the log file format
            log_file.write("# {}\n".format(commit))
            log_file.write(build_times.text)
            log_file.flush()
            # don't fetch another build
            break