gitlab-extract.py 5.09 KB
Newer Older
1
#!/usr/bin/env python3
2
import argparse, pprint, sys, glob, zipfile, subprocess
3
import requests
4
import parse_log
5

6
def last(it):
7
    r = None
8 9 10 11
    for i in it:
        r = i
    return r

12 13 14
def first(it):
    for i in it:
        return i
15
    return None
16 17 18

def req(path):
    url = '%s/api/v3/%s' % (args.server, path)
19 20 21
    r = requests.get(url, headers={'PRIVATE-TOKEN': args.private_token})
    r.raise_for_status()
    return r
22 23

# read command-line arguments
24
parser = argparse.ArgumentParser(description='Extract iris-coq build logs from GitLab')
25 26
parser.add_argument("-t", "--private-token",
                    dest="private_token", required=True,
27
                    help="The private token used to authenticate access.")
28 29
parser.add_argument("-s", "--server",
                    dest="server", default="https://gitlab.mpi-sws.org/",
30
                    help="The GitLab server to contact.")
31
parser.add_argument("-p", "--project",
32
                    dest="project", default="FP/iris-coq",
33 34 35 36 37 38 39
                    help="The name of the project on GitLab.")
parser.add_argument("-f", "--file",
                    dest="file", required=True,
                    help="Filename to store the load in.")
parser.add_argument("-c", "--commits",
                    dest="commits",
                    help="The commits to fetch. Default is everything since the most recent entry in the log file.")
40 41 42
parser.add_argument("-a", "--artifacts",
                    dest="artifacts",
                    help="Location of the artifacts (following GitLab's folder structure).  If not given (which should be the common case), the artifacts will be downloaded from GitLab.")
43 44 45
parser.add_argument("-b", "--blacklist-branch",
                    dest="blacklist_branch",
                    help="Skip the commit if it is contained in the given branch.")
46
args = parser.parse_args()
47
log_file = sys.stdout if args.file == "-" else open(args.file, "a")
48

49 50 51 52
# determine commit, if missing
if args.commits is None:
    if args.file == "-":
        raise Exception("If you do not give explicit commits, you have to give a logfile so that we can determine the missing commits.")
53
    last_result = last(parse_log.parse(open(args.file, "r"), parse_times = parse_log.PARSE_NOT))
54 55
    args.commits = "{}..origin/master".format(last_result.commit)

Ralf Jung's avatar
Ralf Jung committed
56
projects = req("projects?per_page=512")
57
project = first(filter(lambda p: p['path_with_namespace'] == args.project, projects.json()))
58 59 60
if project is None:
    sys.stderr.write("Project not found.\n")
    sys.exit(1)
61

62
BREAK = False
63
for commit in parse_log.parse_git_commits(args.commits):
64 65
    if BREAK:
        break
66 67 68 69 70 71 72
    # test to skip the commit
    if args.blacklist_branch is not None:
        branches = subprocess.check_output(["git", "branch", "-r", "--contains", commit]).decode("utf-8")
        if args.blacklist_branch in map(lambda x: x.strip(), branches.split('\n')):
            continue
    # Find out more about the commit
    print("Fetching {}...".format(commit), end='')
73 74 75
    commit_data = req("/projects/{}/repository/commits/{}".format(project['id'], commit))
    if commit_data.status_code != 200:
        raise Exception("Commit not found?")
76 77
    builds = req("/projects/{}/repository/commits/{}/builds".format(project['id'], commit))
    if builds.status_code != 200:
78 79
        raise Exception("Build not found?")
    # iterate over builds by decreasing ID, and look for the artifact
80
    found_build = False
81 82 83 84
    for build in builds.json():
        if build['status'] in ('created', 'pending', 'running'):
            # build still not yet done, don't fetch this or any later commit
            BREAK = True
85
            print(" build still in progress, aborting")
86 87 88 89 90
            break
        if build['status'] != 'success':
            # build failed or cancelled, skip to next
            continue
        # now fetch the build times
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
        if args.artifacts:
            artifact_zip = glob.glob('{}/*/{}/{}/artifacts.zip'.format(args.artifacts, project['id'], build['id']))
            if not artifact_zip:
                # no artifact at this build, try another one
                continue
            assert len(artifact_zip) == 1, "Found too many artifacts"
            artifact_zip = artifact_zip[0]
            with zipfile.ZipFile(artifact_zip) as artifact:
                with artifact.open('build-time.txt') as build_times:
                    # Output into log file
                    log_file.write("# {}\n".format(commit))
                    log_file.write(build_times.read().decode('UTF-8'))
                    log_file.flush()
        else:
            build_times = requests.get("{}/builds/{}/artifacts/raw/build-time.txt".format(project['web_url'], build['id']))
            if build_times.status_code != 200:
                # no artifact at this build, try another one
                continue
            # Output in the log file format
            log_file.write("# {}\n".format(commit))
            log_file.write(build_times.text)
            log_file.flush()
            # don't fetch another build
114 115
            found_build = True
            print(" success")
116
            break
117
    if not found_build and not BREAK:
118
        print(" found no succeessful build")