updateinfo.py

import glob
import json
import OMPython
import os
import pygit2
import re
import requests
import shutil
import zipfile
from github import Github, Auth
from atlassian import Bitbucket

from ompackagemanager import common

bitbucket = Bitbucket(url="https://api.bitbucket.org")


def removerepo(ghurl, repopath):
    print("Removing repository " + ghurl)
    shutil.rmtree(repopath)


tagregex = re.compile('^refs/tags')


def alltags(gitrepo):
    return list((remove_prefix(e, "refs/tags/"), str(gitrepo.lookup_reference_dwim(e).target))
                for e in filter(lambda r: tagregex.match(r), gitrepo.listall_references()))


def remove_prefix(text, prefix):
    return text[text.startswith(prefix) and len(prefix):]


def allbranches(gitrepo):
    return [(remove_prefix(e, "origin/"), str(gitrepo.lookup_reference_dwim(e).target))
            for e in gitrepo.branches.remote]


def getgitrepo(url, repopath):
    if os.path.exists(repopath):
        gitrepo = pygit2.Repository(repopath)
        if len(gitrepo.remotes) != 1 or gitrepo.remotes[0].url != url:
            removerepo(url, repopath)
        else:
            gitrepo.remotes[0].fetch()
    if not os.path.exists(repopath):
        pygit2.clone_repository(url, repopath)
    gitrepo = pygit2.Repository(repopath)
    return gitrepo


def insensitive_glob(pattern):
    def either(c):
        return '[%s%s]' % (c.lower(), c.upper()) if c.isalpha() else c
    return glob.glob(''.join(either(char) for char in pattern))


def main():
    """Generate up-to-date `rawdata.json` from `repos.json`.

    Queries the repositories where the libraries are stored.
    """

    token = Auth.Token(os.environ["GITHUB_AUTH"])
    github_api = Github(auth=token)

    omc = OMPython.OMCSessionZMQ()

    with open("repos.json", "r") as f:
        data = json.load(f)
    if os.path.exists("rawdata.json"):
        serverdata = json.load(open("rawdata.json"))
    else:
        serverdata = {}

    if not os.path.exists("cache"):
        os.mkdir("cache")

    namesInFile = set()
    for key in data.keys():
        for name in data[key].get("names", []):
            if name in namesInFile:
                raise Exception(key + " exists multiple times")
            namesInFile.add(name)

    for key in sorted(data.keys()):
        entry = data[key]
        repopath = os.path.join("cache", key)
        if "ignore" in entry:
            continue

        if len(insensitive_glob(repopath)) > 1:
            for d in insensitive_glob(repopath):
                print("Cleaning duplicate case-insensitive names")
                shutil.rmtree(d)

        if "github" in entry or "git" in entry or "zipfiles" in entry:
            if "github" in entry:
                try:
                    r = github_api.get_repo(entry["github"])
                    branches = list((b.name, b.commit.sha)
                                    for b in r.get_branches())
                    tags = list((b.name, b.commit.sha) for b in r.get_tags())
                    giturl = "https://github.com/%s.git" % entry["github"]
                except BaseException:
                    print("Failed to get github entry: %s" % entry["github"])
                    raise
            elif "git" in entry:
                giturl = entry["git"]
                gitrepo = getgitrepo(giturl, repopath + ".git")
                branches = allbranches(gitrepo)
                if entry.get("bitbucket-api-downloads-instead-of-tags"):
                    tags = []
                    for download in bitbucket._get_paged(
                        "2.0/repositories/%s/downloads" %
                            entry.get("bitbucket-api-downloads-instead-of-tags")):
                        name = download["name"]
                        if name.endswith(".zip") and name.startswith(key):
                            ver = common.VersionNumber(
                                name[len(key):-4].strip("-").strip(" "))
                            if ver.major == 0 and ver.minor == 0 and ver.patch == 0:
                                continue
                            tags.append(
                                ("v" + str(ver), download["links"]["self"]["href"]))
                else:
                    tags = alltags(gitrepo)
            elif "zipfiles" in entry:
                branches = []
                tags = list(entry["zipfiles"].items())
                giturl = ""

            if key not in serverdata:
                serverdata[key] = {}
                print("Did not have stored data for " + key)
            if "refs" not in serverdata[key]:
                serverdata[key]["refs"] = {}
            ignoreTags = set()
            if "ignore-tags" in entry:
                ignoreTags = set(entry["ignore-tags"])
            objects = []
            for (name, sha) in branches:
                if name in (entry.get("branches") or []):
                    objects.append((entry["branches"][name], sha))
            for (name, sha) in tags:
                if name not in ignoreTags:
                    objects.append((name, sha))

            if not objects:
                raise Exception("No commits or zip-files found for %s" % key)

            tagsDict = serverdata[key]["refs"]

            for (tagName, sha) in objects:
                if not isinstance(tagName, str):
                    names = tagName["names"]
                    tagName = tagName["version"]
                else:
                    names = entry["names"]
                v = common.VersionNumber(tagName)
                # v3.2.1+build.0-beta.1 is not a pre-release...
                for build in v.build:
                    if "-" in build:
                        raise Exception(
                            "Release build string looks like a pre-release: %s" % v)
                if tagName not in tagsDict:
                    tagsDict[tagName] = {}
                thisTagBackup = tagsDict[tagName]
                thisTag = tagsDict[tagName]

                entrykind = "zip" if (
                    "zipfiles" in entry or sha.startswith("http")) else "sha"

                if (entrykind not in thisTag) or (thisTag[entrykind] != sha):
                    if entrykind == "zip":
                        try:
                            os.unlink(repopath)
                        except BaseException:
                            pass
                        try:
                            shutil.rmtree(repopath)
                        except FileNotFoundError:
                            pass
                        os.mkdir(repopath)
                        zipfilepath = repopath + "-" + tagName + ".zip"
                        with open(zipfilepath, 'wb') as fout:
                            fout.write(
                                requests.get(
                                    sha, allow_redirects=True).content)
                        with zipfile.ZipFile(zipfilepath, 'r') as zip_ref:
                            zip_ref.extractall(repopath)
                    else:
                        gitrepo = getgitrepo(giturl, repopath + ".git")
                        try:
                            gitrepo.checkout_tree(
                                gitrepo.get(sha),
                                strategy=pygit2.GIT_CHECKOUT_FORCE | pygit2.GIT_CHECKOUT_RECREATE_MISSING)
                            try:
                                os.unlink(repopath)
                            except BaseException:
                                pass
                            try:
                                shutil.rmtree(repopath)
                            except FileNotFoundError:
                                pass
                            shutil.copytree(repopath + ".git", repopath)
                        except BaseException:
                            print(
                                "Failed to checkout %s from repo: %s with SHA %s" %
                                (tagName, repopath, sha))
                            print(
                                "Usually this error happens if somebody force-pushed to a repository.")
                            print(
                                "We will delete our own cache of %s now!", repopath)
                            print("PLEASE RERUN THIS SCRIPT!")
                            try:
                                os.unlink(repopath)
                            except BaseException:
                                pass
                            try:
                                shutil.rmtree(repopath)
                            except FileNotFoundError:
                                pass
                            raise
                    omc.sendExpression(
                        "OpenModelica.Scripting.getErrorString()")

                    provided = {}
                    for libname in names:
                        hits = insensitive_glob(
                            os.path.join(repopath, "package.mo"))
                        if len(hits) == 1:
                            if libname != entry["names"][0]:
                                continue
                        else:
                            hits = []
                            for extraPath in (
                                    [""] + (entry.get("search-extra-paths") or [])):
                                p = repopath
                                if extraPath != "":
                                    p = os.path.join(repopath, extraPath)
                                else:
                                    p = repopath
                                hitsNew = (
                                    insensitive_glob(
                                        os.path.join(
                                            p,
                                            libname,
                                            "package.mo")) +
                                    insensitive_glob(
                                        os.path.join(
                                            p,
                                            libname +
                                            " *",
                                            "package.mo")) +
                                    insensitive_glob(
                                        os.path.join(
                                            p,
                                            libname +
                                            "-*",
                                            "package.mo")) +
                                    insensitive_glob(
                                        os.path.join(
                                            p,
                                            libname +
                                            ".mo")) +
                                    insensitive_glob(
                                        os.path.join(
                                            p,
                                            libname +
                                            " *.mo")) +
                                    insensitive_glob(
                                        os.path.join(
                                            p,
                                            libname +
                                            "-*.mo")) +
                                    insensitive_glob(
                                        os.path.join(
                                            p,
                                            libname +
                                            "*",
                                            libname +
                                            ".mo")) +
                                    insensitive_glob(
                                        os.path.join(
                                            p,
                                            libname +
                                            "*",
                                            libname +
                                            "-*.mo")) +
                                    insensitive_glob(
                                        os.path.join(
                                            p,
                                            libname +
                                            "*",
                                            libname +
                                            " *.mo")))
                                hits += hitsNew
                        if len(hits) != 1:
                            print(str(len(hits)) + " hits for " +
                                  libname + " in " + tagName + ": " + str(hits))
                            print(
                                "Check that " +
                                libname +
                                "/package.mo exists in tag: " +
                                tagName +
                                ". Maybe wrong spelling? If different naming than the repo add it to the 'names' for the lib in repos.json")
                            continue
                        omc.sendExpression("clear()")
                        if "standard" in entry:
                            grammar = common.findMatchingLevel(
                                tagName, entry["standard"])
                            if grammar is None:
                                grammar = "latest"
                        else:
                            grammar = "latest"
                        omc.sendExpression(
                            "setCommandLineOptions(\"--std=%s\")" % grammar)

                        if not omc.sendExpression(
                            "loadFile(\"%s\", uses=false)" %
                                hits[0]):
                            print(
                                "Failed to load file %s in %s" %
                                (hits[0], tagName))
                            continue
                        classNamesAfterLoad = omc.sendExpression(
                            "getClassNames()")
                        if libname not in classNamesAfterLoad:
                            print("Did not load the library? ")
                            print(classNamesAfterLoad)
                            continue
                        version = omc.sendExpression(
                            "getVersion(%s)" % libname)
                        if version == "":
                            version = str(common.VersionNumber(tagName))
                        else:
                            v1 = common.VersionNumber(version)
                            v2 = common.VersionNumber(tagName)
                            if len(v2.prerelease) == 0 and entry.get("semverTagOverridesAnnotation") and (
                                    v2 > v1 or entry["semverTagOverridesAnnotation"] == "alsoNewerVersions"):
                                v1 = v2
                            if len(v2.prerelease) > 0 and (len(v1.prerelease) == 0 or entry.get(
                                    "semverPrereleaseOverridesAnnotation") or tagName in ["master", "main", "trunk"]):
                                v1.prerelease = v2.prerelease
                                if tagName in [
                                        "master", "main", "trunk"] and len(
                                        v2.build) == 0:
                                    v1.build = []
                            if v1.major == v2.major and v1.minor == v2.minor and v1.patch == v2.patch and len(
                                    v1.prerelease) == 0 and len(v1.build) == 0:
                                version = str(v2)
                            else:
                                version = str(v1)
                        if version.startswith("0.0.0-"):
                            version = version[6:]
                        if version.startswith("+"):
                            version = version[1:]
                        uses = sorted([[e[0], str(common.VersionNumber(e[1]))]
                                      for e in omc.sendExpression("getUses(%s)" % libname)])
                        for ignore in entry.get("ignore-uses", []):
                            uses = [use for use in uses if use[0] != ignore]
                        # Get conversions
                        (withoutConversion, withConversion) = omc.sendExpression(
                            "getConversionsFromVersions(%s)" % libname)
                        withoutConversion = list(filter(None, [str(ver) for ver in sorted(
                            [common.VersionNumber(v) for v in withoutConversion])]))
                        withConversion = list(filter(None, [str(ver) for ver in sorted(
                            [common.VersionNumber(v) for v in withConversion])]))
                        path = hits[0][len(repopath) + 1:]
                        if os.path.basename(hits[0]) == "package.mo":
                            path = os.path.dirname(path)
                        libentry = {"version": version, "path": path}
                        if len(uses) > 0:
                            libentry["uses"] = dict(uses)
                        if len(withoutConversion) > 0:
                            libentry["provides"] = withoutConversion
                        if len(withConversion) > 0:
                            libentry["convertFromVersion"] = withConversion
                        provided[libname] = libentry
                    try:
                        errorString = omc.sendExpression(
                            "OpenModelica.Scripting.getErrorString()")
                    except UnicodeDecodeError:
                        print("UnicodeDecodeError for %s %s" % (key, tagName))
                        raise
                    if len(provided) == 0:
                        # + ":" + errorString)
                        print("Broken for " + key + " " + tagName)
                        tagsDict[tagName] = thisTagBackup
                        continue
                    thisTag["libs"] = provided
                    thisTag[entrykind] = sha
                    if "broken" in thisTag:
                        del thisTag["broken"]
                # level = getSupportLevel(tagName, entry["support"])
                # thisTag["support"] = level
            serverdata[key]["refs"] = tagsDict
        else:
            raise Exception(
                "Don't know how to handle entry for %s: %s" %
                (key, entry))
    with open("rawdata.json", "w") as io:
        json.dump(serverdata, io, sort_keys=True, indent=2)