Merge remote-tracking branch 'qmk/master' into merge-2024-09-07

util/ci/discord-results.py

@@ -11,6 +11,7 @@ parser = argparse.ArgumentParser(prog='discord-results.py', description='Sends a
 parser.add_argument('-b', '--branch')
 parser.add_argument('-k', '--keymap')
 parser.add_argument('-u', '--url')
+parser.add_argument('-s', '--sha')
 args = parser.parse_args()
 
 qmk_dir = Path(__file__).resolve().parents[2].resolve()
@@ -43,6 +44,7 @@ else:
 
 embed.add_embed_field(name='Build Target', value=f'[**{args.branch}**](https://github.com/qmk/qmk_firmware/tree/{args.branch}) / **{args.keymap}** keymap')
 embed.add_embed_field(name='Workflow Run', value=f'[**Link**]({args.url})')
+embed.add_embed_field(name='Firmware Binaries', value=f'[**ci.qmk.fm**](https://ci.qmk.fm/{args.branch}/{args.sha}/index.html)')
 embed.set_timestamp()
 
 webhook.add_embed(embed)

util/ci/firmware_list_generator.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import os
+import json
+from pathlib import Path
+from time import gmtime, strftime
+
+DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
+
+
+def current_datetime():
+    return strftime(DATETIME_FORMAT, gmtime())
+
+
+qmk_firmware_dir = Path(os.path.realpath(__file__)).parents[2]
+
+binaries = []
+binaries.extend(qmk_firmware_dir.glob("*.bin"))
+binaries.extend(qmk_firmware_dir.glob("*.hex"))
+binaries.extend(qmk_firmware_dir.glob("*.uf2"))
+binaries = list(sorted(binaries))
+
+data = []
+for binary in binaries:
+    data.append(binary.name)
+
+keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'files': data}, separators=(',', ':'))
+
+print(keyboard_all_json)

util/ci/index_generator.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import shlex
+import subprocess
+from pathlib import Path
+
+from ansi2html import Ansi2HTMLConverter
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+orig_cwd = os.getcwd()
+qmk_firmware_dir = Path(os.path.realpath(__file__)).parents[2]
+build_dir = qmk_firmware_dir / ".build"
+
+KEYBOARD_PATTERN = re.compile("CI Metadata: KEYBOARD=(?P<keyboard>.*)\r?\n")
+KEYMAP_PATTERN = re.compile("CI Metadata: KEYMAP=(?P<keymap>.*)\r?\n")
+
+env = Environment(
+    loader=FileSystemLoader(Path(os.path.realpath(__file__)).parent / "templates"),
+    autoescape=select_autoescape(),
+)
+
+
+def _run(command, capture_output=True, combined_output=False, text=True, **kwargs):
+    if isinstance(command, str):
+        command = shlex.split(command)
+    if capture_output:
+        kwargs["stdout"] = subprocess.PIPE
+        kwargs["stderr"] = subprocess.PIPE
+    if combined_output:
+        kwargs["stderr"] = subprocess.STDOUT
+    if "stdin" in kwargs and kwargs["stdin"] is None:
+        del kwargs["stdin"]
+    if text:
+        kwargs["universal_newlines"] = True
+    return subprocess.run(command, **kwargs)
+
+
+def _ansi2html(value):
+    return Ansi2HTMLConverter().convert(value, full=False)
+
+
+def _ansi2html_styles():
+    from ansi2html.style import get_styles
+
+    styles = get_styles(scheme="dracula")
+    return "\n".join([str(s) for s in styles])
+
+
+def _git_log(count=4):
+    os.chdir(qmk_firmware_dir)
+    ret = _run(f"git log -n {count} --color=always --no-merges --topo-order --stat").stdout.strip()
+    os.chdir(orig_cwd)
+    return ret
+
+
+def _git_describe():
+    os.chdir(qmk_firmware_dir)
+    ret = _run("git describe --tags --always --dirty").stdout.strip()
+    os.chdir(orig_cwd)
+    return ret
+
+
+def _git_revision():
+    os.chdir(qmk_firmware_dir)
+    ret = _run("git rev-parse HEAD").stdout.strip()
+    os.chdir(orig_cwd)
+    return ret
+
+
+env.filters["ansi2html"] = _ansi2html
+
+binaries = []
+binaries.extend(qmk_firmware_dir.glob("*.bin"))
+binaries.extend(qmk_firmware_dir.glob("*.hex"))
+binaries.extend(qmk_firmware_dir.glob("*.uf2"))
+binaries = list(sorted(binaries))
+
+failures = []
+for mdfile in list(sorted(build_dir.glob("failed.log.*"))):
+    txt = Path(mdfile).read_text()
+
+    m_kb = KEYBOARD_PATTERN.search(txt)
+    if not m_kb:
+        raise Exception("Couldn't determine the keyboard from the failure output")
+    m_km = KEYMAP_PATTERN.search(txt)
+    if not m_km:
+        raise Exception("Couldn't determine the keymap from the failure output")
+
+    txt = KEYBOARD_PATTERN.sub("", KEYMAP_PATTERN.sub("", txt)).strip()
+
+    failures.append({
+        "stdout": txt,
+        "keyboard": m_kb.group("keyboard"),
+        "keymap": m_km.group("keymap"),
+    })
+
+template = env.get_template("index.html.j2")
+print(template.render(
+    ansi2html_styles=_ansi2html_styles(),
+    git_log=_git_log(),
+    git_describe=_git_describe(),
+    git_revision=_git_revision(),
+    binaries=binaries,
+    failures=failures,
+))

util/ci/requirements.txt

@@ -1 +1,3 @@
 discord-webhook
+Jinja2
+ansi2html

util/ci/templates/index.html.j2

@@ -0,0 +1,140 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="icon" href="https://qmk.fm/logo.png" />
+    <style type="text/css">
+        {{ ansi2html_styles }}
+
+        body {
+            color: #FFF;
+            background-color: #111;
+        }
+
+        h3 {
+            font-family: sans-serif;
+            margin-top: 0;
+        }
+
+        a {
+            color: #00e1ff;
+            text-decoration: none;
+        }
+
+        a:hover {
+            color: #f700ff;
+            text-decoration: underline;
+        }
+
+        a:visited {
+            color: #00e1ff;
+        }
+
+        .build-target {
+            background-color: #333;
+            font-family: monospace;
+            padding-left: 0.3em;
+            padding-right: 0.3em;
+        }
+
+        .binary-link {
+            margin: 0;
+            margin-top: 0.25em;
+        }
+
+        .container {
+            background-color: #222;
+            padding: 2.0em;
+            margin: 2.0em;
+            border-radius: 2.0em;
+        }
+
+        .header-container {
+            display: table-cell;
+            padding-left: 1.5em;
+            vertical-align: middle;
+            font-family: sans-serif;
+        }
+
+        .header-container div:not(:first-child) {
+            margin-top: 0.25em;
+        }
+    </style>
+</head>
+
+<body class="" style="font-size: normal;">
+    <div style="float: left">
+        <div class="container">
+            <div style="display: table-cell; vertical-align: middle;">
+                <a href="https://qmk.fm/"><img src="https://qmk.fm/badge-community-dark.svg" style="width: 30em;" /></a>
+            </div>
+            <div class="header-container">
+                <div>
+                    <span style="font-size: 175%; font-weight: bold;">CI Build {% if failures | length > 0 %}&#10060;{% else %}&#9989;{% endif %}</span>
+                </div>
+                <div>
+                    <span style="font-size: 100%; font-family: monospace;">{{ git_describe }}</span>
+                </div>
+                <div>
+                    <span style="font-size: 100%; font-family: monospace;">{{ git_revision }}</span>
+                </div>
+                <div>
+                    <span style="font-size: 80%; font-weight: bold; color: {% if failures | length > 0 %}#F00{% else %}#0F0{% endif %};">{{ failures | length }} failure{% if failures | length != 1 %}s{% endif %}</span>
+                </div>
+                <div>
+                    <span style="font-size: 80%">
+                    {% if binaries | length > 0 %}<a href="#firmwares">Firmwares</a> | {% endif %}
+                    <a href="#commit_info">Commit info</a>
+                    {% if failures | length > 0 %} | <a href="#failure_logs">Failure Logs</a>{% endif %}
+                    </span>
+                </div>
+            </div>
+        </div>
+        <a name="commit_info"></a>
+        <div class="container">
+            <h3>Git commit</h3>
+            <div class="body_foreground body_background"
+                style="display: table-cell; padding: 1.0em; border-radius: 1.0em;">
+                <pre class="ansi2html-content">{{ git_log | ansi2html }}</pre>
+            </div>
+        </div>
+        {% if failures | length > 0 %}
+        <a name="failure_logs"></a>
+        <div class="container">
+            <h3>Build failure logs</h3>
+            <ul>
+                {% for failure in failures %}
+                <li><a style="font-family: monospace;" href="#build_failure_{{ failure.keyboard }}_{{ failure.keymap }}">{{ failure.keyboard }}:{{ failure.keymap }}</a></li>
+                {% endfor %}
+            </ul>
+        </div>
+        {% for failure in failures %}
+        <a name="build_failure_{{ failure.keyboard }}_{{ failure.keymap }}"></a>
+        <div class="container">
+            <h3>Build failure &mdash; <span class="build-target">{{ failure.keyboard }}:{{ failure.keymap }}</span></h3>
+            <div class="body_foreground body_background"
+                style="display: table-cell; padding: 1.0em; border-radius: 1.0em;">
+                <pre class="ansi2html-content">{{ failure.stdout | ansi2html }}</pre>
+            </div>
+        </div>
+        {% endfor %}
+        {% endif %}
+    </div>
+    {% if binaries | length > 0 %}
+    <div style="float: right">
+        <a name="firmwares"></a>
+        <div class="container">
+            <h3>Firmware downloads</h3>
+            <div class="body_foreground body_background"
+                style="display: table-cell; padding: 1.0em; border-radius: 1.0em;">
+                {% for binary in binaries %}
+                <p class="binary-link" style="font-family: monospace;"><a href="{{ binary.name }}">{{ binary.name }}</a></p>
+                {%- endfor %}
+            </div>
+        </div>
+    </div>
+    {% endif %}
+</body>
+
+</html>

util/install/fedora.sh

@@ -4,7 +4,7 @@ _qmk_install() {
     echo "Installing dependencies"
 
     . /etc/os-release
-    if [ "$VERSION_ID" == "39" ]; then
+    if [ "$VERSION_ID" -ge "39" ]; then
         sudo dnf $SKIP_PROMPT copr enable erovia/dfu-programmer
     fi
 

util/update_chibios_mirror.sh

@@ -4,13 +4,10 @@
 # Configuration
 
 # The ChibiOS branches to mirror
-chibios_branches="trunk stable_20.3.x stable_21.11.x"
-
-# The ChibiOS tags to mirror
-chibios_tags="ver20.3.1 ver20.3.2 ver20.3.3 ver20.3.4 ver21.11.1 ver21.11.2 ver21.11.3"
+chibios_branches="trunk stable_21.11.x"
 
 # The ChibiOS-Contrib branches to mirror
-contrib_branches="chibios-20.3.x chibios-21.11.x"
+contrib_branches="chibios-21.11.x"
 
 ################################
 # Actions
@@ -46,6 +43,12 @@ fi
 echo "Updating remotes..."
 git fetch --all --tags --prune
 
+echo "Ensure refs actually match up..."
+for branch in $chibios_branches ; do
+    echo "Matching $branch..."
+    git update-ref refs/remotes/svn/$branch refs/remotes/qmk/svn-mirror/$branch
+done
+
 echo "Fetching latest from subversion..."
 git svn fetch
 
@@ -56,13 +59,6 @@ for branch in $chibios_branches ; do
         && git push qmk svn-mirror/$branch
 done
 
-echo "Updating ChibiOS tags..."
-for tagname in $chibios_tags ; do
-    echo "Creating tag 'svn-mirror/$tagname' from 'svn/tags/$tagname'..."
-    GIT_COMMITTER_DATE="$(git log -n1 --pretty=format:'%ad' svn/tags/$tagname)" git tag -f -a -m "Tagging $tagname" svn-mirror/$tagname svn/tags/$tagname
-    git push qmk svn-mirror/$tagname
-done
-
 cd "$contrib_dir"
 
 if [[ -z "$(cat "$contrib_git_config" | grep '\[remote "qmk"\]')" ]] ; then