fix(ruff): update Python code to conform to 0.16

This commit is contained in:
Jonathan Steffan authored and xytovl committed 2026-07-24 21:14:49 +02:00
1 parent 8b54ba1643
commit 00d2727049
11 files changed
+465 -396

No files matched your search

+2
View File
@@ -32,8 +32,10 @@ jobs:
- name: Run ruff lint check
uses: astral-sh/ruff-action@v3
with:
version: "0.16.0"
args: "check"
- name: Run ruff format check
uses: astral-sh/ruff-action@v3
with:
version: "0.16.0"
args: "format --check"
+1 -1
View File
@@ -1,6 +1,6 @@
# Ruff config — https://docs.astral.sh/ruff/
# `include` sets which files are linted; widen it to cover more paths.
include = ["tools/perfetto/**/*.py"]
include = ["tools/**/*.py"]
line-length = 100
target-version = "py310"
+30 -28
View File
@@ -1,47 +1,49 @@
#!/usr/bin/env python3
import os
import re
import subprocess
import sys
from elftools.elf.elffile import ELFFile
from elftools.elf.sections import NoteSection
import os
import sys
import subprocess
def get_lib_list(path):
def get_build_id(filename):
with open(filename, 'rb') as f:
for sect in ELFFile(f).iter_sections():
if not isinstance(sect, NoteSection):
continue
for note in sect.iter_notes():
if note['n_type'] in {'NT_GNU_BUILD_ID', 'NT_GNU_GOLD_VERSION'}:
return note['n_desc']
return None
def get_build_id(filename):
with open(filename, "rb") as f:
for sect in ELFFile(f).iter_sections():
if not isinstance(sect, NoteSection):
continue
for note in sect.iter_notes():
if note["n_type"] in {"NT_GNU_BUILD_ID", "NT_GNU_GOLD_VERSION"}:
return note["n_desc"]
return None
libs = {}
libs = {}
for root, dirs, files in os.walk(path):
for name in files:
if name.endswith('.so'):
build_id = get_build_id(os.path.join(root, name))
if build_id:
libs[build_id] = os.path.join(root, name)
for root, dirs, files in os.walk(path):
for name in files:
if name.endswith(".so"):
build_id = get_build_id(os.path.join(root, name))
if build_id:
libs[build_id] = os.path.join(root, name)
return libs
return libs
libs = get_lib_list('build/intermediates/merged_native_libs')
libs = get_lib_list("build/intermediates/merged_native_libs")
r = re.compile("^.*#[0-9]+ pc ([0-9a-f]+) .*BuildId: ([0-9a-f]*).*")
for line in sys.stdin:
m = r.match(line.rstrip())
if m:
address = m[1]
build_id = m[2]
m = r.match(line.rstrip())
if m:
address = m[1]
build_id = m[2]
if build_id in libs:
lib = libs[build_id]
subprocess.run(['llvm-symbolizer'], input=(lib + ' 0x' + address).encode("utf-8"))
if build_id in libs:
lib = libs[build_id]
subprocess.run(
["llvm-symbolizer"], input=(lib + " 0x" + address).encode("utf-8"), check=False
)
+124 -106
View File
@@ -1,133 +1,151 @@
#!/usr/bin/env python
import polib
import argparse
import os
import pathlib
import subprocess
import tempfile
def list_files(base: str, exts: str|list[str]):
if type(exts) == str:
exts = [exts]
for root, dirs, files in os.walk(base):
for f in files:
for ext in exts:
if f.endswith(ext):
yield os.path.join(root, f)
import polib
ISSUE_LABEL="localisation"
ISSUE_TITLE="[{lang}] Missing translations"
def list_files(base: str, exts: str | list[str]):
if type(exts) == str:
exts = [exts]
for root, dirs, files in os.walk(base):
for f in files:
for ext in exts:
if f.endswith(ext):
yield os.path.join(root, f)
ISSUE_LABEL = "localisation"
ISSUE_TITLE = "[{lang}] Missing translations"
if __name__ == "__main__":
root = os.path.dirname(os.path.dirname(__file__))
locale_dir = os.path.join(root, "locale")
cwd = os.getcwd()
root = os.path.dirname(os.path.dirname(__file__))
locale_dir = os.path.join(root, "locale")
cwd = os.getcwd()
parser = argparse.ArgumentParser(
prog='check_po.py',
description='Check that all strings are translated in a .po file')
parser = argparse.ArgumentParser(
prog="check_po.py", description="Check that all strings are translated in a .po file"
)
parser.add_argument('lang', type=str, nargs="*")
parser.add_argument('--github-repository')
parser.add_argument('--github-token')
parser.add_argument('--manage-issues', action="store_true")
parser.add_argument("lang", type=str, nargs="*")
parser.add_argument("--github-repository")
parser.add_argument("--github-token")
parser.add_argument("--manage-issues", action="store_true")
args = parser.parse_args()
args = parser.parse_args()
if len(args.lang) == 0:
for d in sorted(os.listdir(locale_dir)):
if os.path.isdir(os.path.join(locale_dir, d)):
args.lang.append(d)
if len(args.lang) == 0:
for d in sorted(os.listdir(locale_dir)):
if os.path.isdir(os.path.join(locale_dir, d)):
args.lang.append(d)
if args.manage_issues:
from github import Auth, Github, GithubException
gh = Github(auth=Auth.Token(args.github_token))
repo = gh.get_repo(args.github_repository)
if args.manage_issues:
from github import Auth, Github, GithubException
with tempfile.TemporaryDirectory() as tmpdir:
client_pot = os.path.join(tmpdir, "client.pot")
dashboard_pot = os.path.join(tmpdir, "dashboard.pot")
gh = Github(auth=Auth.Token(args.github_token))
repo = gh.get_repo(args.github_repository)
subprocess.check_call(
["xgettext",
"--c++", "--from-code=UTF-8",
"--keyword=_:1,1t",
"--keyword=_S:1,1t",
"--keyword=_C:1c,2,2t",
"--keyword=_cS:1c,2,2t",
"--keyword=_cF:1c,2,2t",
"--keyword=_F:1,1t",
"--keyword=gettext_noop:1,1t",
"--output", client_pot,
"--package-name=WiVRn",
] + sorted(list_files(os.path.join(root, "client"), ".cpp"))
)
with tempfile.TemporaryDirectory() as tmpdir:
client_pot = os.path.join(tmpdir, "client.pot")
dashboard_pot = os.path.join(tmpdir, "dashboard.pot")
subprocess.check_call(
["xgettext",
"--c++", "--kde", "--from-code=UTF-8",
"--keyword=i18n:1",
"--keyword=i18nc:1c,2",
"--keyword=i18np:1,2",
"--keyword=i18ncp:1c,2,3",
"--output", dashboard_pot,
"--package-name=WiVRn-dashboard",
] + sorted(list_files(os.path.join(root, "dashboard"), [".cpp", ".qml"]))
)
subprocess.check_call(
[
"xgettext",
"--c++",
"--from-code=UTF-8",
"--keyword=_:1,1t",
"--keyword=_S:1,1t",
"--keyword=_C:1c,2,2t",
"--keyword=_cS:1c,2,2t",
"--keyword=_cF:1c,2,2t",
"--keyword=_F:1,1t",
"--keyword=gettext_noop:1,1t",
"--output",
client_pot,
"--package-name=WiVRn",
]
+ sorted(list_files(os.path.join(root, "client"), ".cpp"))
)
client_ref = polib.pofile(client_pot)
dashboard_ref = polib.pofile(dashboard_pot)
subprocess.check_call(
[
"xgettext",
"--c++",
"--kde",
"--from-code=UTF-8",
"--keyword=i18n:1",
"--keyword=i18nc:1c,2",
"--keyword=i18np:1,2",
"--keyword=i18ncp:1c,2,3",
"--output",
dashboard_pot,
"--package-name=WiVRn-dashboard",
]
+ sorted(list_files(os.path.join(root, "dashboard"), [".cpp", ".qml"]))
)
for lang in args.lang:
lang_issues = dict()
for po_ref, po in ((client_ref, os.path.join(locale_dir, lang, "wivrn.po")),
(dashboard_ref, os.path.join(locale_dir, lang, "wivrn-dashboard.po"))):
if not os.path.exists(po):
continue
po_file = polib.pofile(po)
po = os.path.relpath(po, cwd)
entries = [i.msgid for i in po_file.translated_entries()]
client_ref = polib.pofile(client_pot)
dashboard_ref = polib.pofile(dashboard_pot)
missing = 0
for i in po_ref:
if not i.obsolete:
if not i.msgid in entries:
missing = missing + 1
print(f"::notice file={po}::{lang} translation for {repr(i.msgid)} is missing")
for lang in args.lang:
lang_issues = {}
for po_ref, po in (
(client_ref, os.path.join(locale_dir, lang, "wivrn.po")),
(dashboard_ref, os.path.join(locale_dir, lang, "wivrn-dashboard.po")),
):
if not os.path.exists(po):
continue
po_file = polib.pofile(po)
po = os.path.relpath(po, cwd)
entries = [i.msgid for i in po_file.translated_entries()]
if not po in lang_issues:
lang_issues[po] = []
lang_issues[po].append(repr(i.msgid))
missing = 0
for i in po_ref:
if not i.obsolete and i.msgid not in entries:
missing = missing + 1
print(f"::notice file={po}::{lang} translation for {i.msgid!r} is missing")
if missing > 0:
print(f"::warning file={po}::{lang} {missing} translations missing")
if po not in lang_issues:
lang_issues[po] = []
lang_issues[po].append(repr(i.msgid))
if args.manage_issues:
issues = [i for i in repo.get_issues(state="all", labels = [ISSUE_LABEL]) if i.title == ISSUE_TITLE.format(lang=lang)]
if missing > 0:
print(f"::warning file={po}::{lang} {missing} translations missing")
issue_body = 'The following translations are missing:\n'
for po in lang_issues:
issue_body += '\n## ' + pathlib.Path(po).name + '\n'
for msg in lang_issues[po]:
issue_body += '- ' + msg + '\n'
if args.manage_issues:
issues = [
i
for i in repo.get_issues(state="all", labels=[ISSUE_LABEL])
if i.title == ISSUE_TITLE.format(lang=lang)
]
if len(lang_issues) > 0:
if issues:
for issue in issues:
if issue.state != "open" or issue.body != issue_body:
issue.edit(state="open", body=issue_body)
else:
try:
repo.create_issue(
title=ISSUE_TITLE.format(lang=lang),
body=issue_body,
labels=[ISSUE_LABEL]
)
except GithubException as e:
# Ignore error when issues are disabled on repository
if e.status != 410:
raise
else:
for issue in issues:
issue.edit(state="closed", body="All translations are up to date")
issue_body = "The following translations are missing:\n"
for po, msgs in lang_issues.items():
issue_body += "\n## " + pathlib.Path(po).name + "\n"
for msg in msgs:
issue_body += "- " + msg + "\n"
if len(lang_issues) > 0:
if issues:
for issue in issues:
if issue.state != "open" or issue.body != issue_body:
issue.edit(state="open", body=issue_body)
else:
try:
repo.create_issue(
title=ISSUE_TITLE.format(lang=lang),
body=issue_body,
labels=[ISSUE_LABEL],
)
except GithubException as e:
# Ignore error when issues are disabled on repository
if e.status != 410:
raise
else:
for issue in issues:
issue.edit(state="closed", body="All translations are up to date")
+10 -9
View File
@@ -1,25 +1,26 @@
#!/usr/bin/env python3
import sys
import os.path
import sys
print(
"""// Generated file, do not edit
"""// Generated file, do not edit
#include <map>
#include <string>
#include <vector>
std::map<std::string, std::vector<int>> glyph_set_per_language = {""")
std::map<std::string, std::vector<int>> glyph_set_per_language = {"""
)
for po_file in sys.argv[1:]:
lang = os.path.basename(os.path.dirname(po_file))
lang = os.path.basename(os.path.dirname(po_file))
with open(po_file) as f:
charset = sorted(set(f.read()))
with open(po_file) as f:
charset = sorted(set(f.read()))
# Basic Latin and Latin-1 Supplement are always included
charset = ', '.join([str(ord(c)) for c in charset if ord(c) > 255])
# Basic Latin and Latin-1 Supplement are always included
charset = ", ".join([str(ord(c)) for c in charset if ord(c) > 255])
print("\t{{ \"{}\", {{{}}}}},".format(lang, charset))
print(f'\t{{ "{lang}", {{{charset}}}}},')
print("};")
+206 -186
View File
@@ -1,29 +1,31 @@
#!/usr/bin/env python3
__license__ = 'MIT'
import json
from urllib.parse import urlparse, ParseResult, parse_qs
import os
__license__ = "MIT"
import argparse
import asyncio
import contextlib
import copy
import glob
import subprocess
import argparse
import logging
import hashlib
import asyncio
from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypedDict
import itertools
import json
import logging
import os
import subprocess
from typing import Any, NamedTuple, TypedDict
from urllib.parse import ParseResult, parse_qs, urlparse
import aiohttp
import toml
CRATES_IO = 'https://static.crates.io/crates'
CARGO_HOME = 'cargo'
CARGO_CRATES = f'{CARGO_HOME}/vendor'
VENDORED_SOURCES = 'vendored-sources'
GIT_CACHE = 'flatpak-cargo/git'
CRATES_IO = "https://static.crates.io/crates"
CARGO_HOME = "cargo"
CARGO_CRATES = f"{CARGO_HOME}/vendor"
VENDORED_SOURCES = "vendored-sources"
GIT_CACHE = "flatpak-cargo/git"
COMMIT_LEN = 7
logger = logging.getLogger(__name__)
@contextlib.contextmanager
def workdir(path: str):
@@ -36,134 +38,141 @@ def workdir(path: str):
def canonical_url(url: str) -> ParseResult:
'Converts a string to a Cargo Canonical URL, as per https://github.com/rust-lang/cargo/blob/35c55a93200c84a4de4627f1770f76a8ad268a39/src/cargo/util/canonical_url.rs#L19'
"Converts a string to a Cargo Canonical URL, as per https://github.com/rust-lang/cargo/blob/35c55a93200c84a4de4627f1770f76a8ad268a39/src/cargo/util/canonical_url.rs#L19"
# Hrm. The upstream cargo does not replace those URLs, but if we don't then it doesn't work too well :(
url = url.replace('git+https://', 'https://')
url = url.replace("git+https://", "https://")
u = urlparse(url)
# It seems cargo drops query and fragment
u = ParseResult(u.scheme, u.netloc, u.path, '', '', '')
u = u._replace(path = u.path.rstrip('/'))
u = ParseResult(u.scheme, u.netloc, u.path, "", "", "")
u = u._replace(path=u.path.rstrip("/"))
if u.netloc == 'github.com':
u = u._replace(scheme = 'https')
u = u._replace(path = u.path.lower())
if u.netloc == "github.com":
u = u._replace(scheme="https")
u = u._replace(path=u.path.lower())
if u.path.endswith('.git'):
u = u._replace(path = u.path[:-len('.git')])
if u.path.endswith(".git"):
u = u._replace(path=u.path[: -len(".git")])
return u
def get_git_tarball(repo_url: str, commit: str) -> str:
url = canonical_url(repo_url)
path = url.path.split('/')[1:]
path = url.path.split("/")[1:]
assert len(path) == 2
owner = path[0]
if path[1].endswith('.git'):
repo = path[1].replace('.git', '')
if path[1].endswith(".git"):
repo = path[1].replace(".git", "")
else:
repo = path[1]
if url.hostname == 'github.com':
return f'https://codeload.{url.hostname}/{owner}/{repo}/tar.gz/{commit}'
elif url.hostname.split('.')[0] == 'gitlab': # type: ignore
return f'https://{url.hostname}/{owner}/{repo}/-/archive/{commit}/{repo}-{commit}.tar.gz'
elif url.hostname == 'bitbucket.org':
return f'https://{url.hostname}/{owner}/{repo}/get/{commit}.tar.gz'
if url.hostname == "github.com":
return f"https://codeload.{url.hostname}/{owner}/{repo}/tar.gz/{commit}"
elif url.hostname.split(".")[0] == "gitlab": # type: ignore
return f"https://{url.hostname}/{owner}/{repo}/-/archive/{commit}/{repo}-{commit}.tar.gz"
elif url.hostname == "bitbucket.org":
return f"https://{url.hostname}/{owner}/{repo}/get/{commit}.tar.gz"
else:
raise ValueError(f'Don\'t know how to get tarball for {repo_url}')
raise ValueError(f"Don't know how to get tarball for {repo_url}")
async def get_remote_sha256(url: str) -> str:
logging.info(f"started sha256({url})")
logger.info(f"started sha256({url})")
sha256 = hashlib.sha256()
async with aiohttp.ClientSession(raise_for_status=True) as http_session:
async with http_session.get(url) as response:
while True:
data = await response.content.read(4096)
if not data:
break
sha256.update(data)
logging.info(f"done sha256({url})")
async with (
aiohttp.ClientSession(raise_for_status=True) as http_session,
http_session.get(url) as response,
):
while True:
data = await response.content.read(4096)
if not data:
break
sha256.update(data)
logger.info(f"done sha256({url})")
return sha256.hexdigest()
_TomlType = Dict[str, Any]
_TomlType = dict[str, Any]
def load_toml(tomlfile: str = 'Cargo.lock') -> _TomlType:
with open(tomlfile, 'r') as f:
def load_toml(tomlfile: str = "Cargo.lock") -> _TomlType:
with open(tomlfile, "r") as f:
toml_data = toml.load(f)
return toml_data
def git_repo_name(git_url: str, commit: str) -> str:
name = canonical_url(git_url).path.split('/')[-1]
return f'{name}-{commit[:COMMIT_LEN]}'
name = canonical_url(git_url).path.split("/")[-1]
return f"{name}-{commit[:COMMIT_LEN]}"
def fetch_git_repo(git_url: str, commit: str) -> str:
repo_dir = git_url.replace('://', '_').replace('/', '_')
cache_dir = os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
clone_dir = os.path.join(cache_dir, 'flatpak-cargo', repo_dir)
if not os.path.isdir(os.path.join(clone_dir, '.git')):
subprocess.run(['git', 'clone', '--depth=1', git_url, clone_dir], check=True)
rev_parse_proc = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=clone_dir, check=True,
stdout=subprocess.PIPE)
repo_dir = git_url.replace("://", "_").replace("/", "_")
cache_dir = os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache"))
clone_dir = os.path.join(cache_dir, "flatpak-cargo", repo_dir)
if not os.path.isdir(os.path.join(clone_dir, ".git")):
subprocess.run(["git", "clone", "--depth=1", git_url, clone_dir], check=True)
rev_parse_proc = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=clone_dir, check=True, stdout=subprocess.PIPE
)
head = rev_parse_proc.stdout.decode().strip()
if head[:COMMIT_LEN] != commit[:COMMIT_LEN]:
subprocess.run(['git', 'fetch', 'origin', commit], cwd=clone_dir, check=True)
subprocess.run(['git', 'checkout', commit], cwd=clone_dir, check=True)
subprocess.run(["git", "fetch", "origin", commit], cwd=clone_dir, check=True)
subprocess.run(["git", "checkout", commit], cwd=clone_dir, check=True)
# Get the submodules as they might contain dependencies. This is a noop if
# there are no submodules in the repository
subprocess.run(['git', 'submodule', 'update', '--init', '--recursive'], cwd=clone_dir, check=True)
subprocess.run(
["git", "submodule", "update", "--init", "--recursive"], cwd=clone_dir, check=True
)
return clone_dir
def update_workspace_keys(pkg, workspace):
for key, item in pkg.items():
# There cannot be a 'workspace' key if the item is not a dict.
if not isinstance(item, dict):
continue;
continue
# Recurse for keys under target.cfg(..)
if key == 'target':
if key == "target":
for target in item.values():
update_workspace_keys(target, workspace)
continue;
continue
# dev-dependencies and build-dependencies should reference root dependencies table from workspace
elif key == 'dev-dependencies' or key == 'build-dependencies':
update_workspace_keys(item, workspace.get('dependencies', None))
continue;
elif key == "dev-dependencies" or key == "build-dependencies":
update_workspace_keys(item, workspace.get("dependencies", None))
continue
if not workspace or not key in workspace:
continue;
continue
workspace_item = workspace[key]
if 'workspace' in item:
if "workspace" in item:
if isinstance(workspace_item, dict):
del item['workspace']
del item["workspace"]
for dep_key, workspace_value in workspace_item.items():
# features are additive
if dep_key == 'features' and 'features' in item:
item['features'] += workspace_value
if dep_key == "features" and "features" in item:
item["features"] += workspace_value
else:
item[dep_key] = workspace_value
elif len(item) > 1:
del item['workspace']
item.update({ 'version': workspace_item })
del item["workspace"]
item.update({"version": workspace_item})
else:
pkg[key] = workspace_item
else:
update_workspace_keys(item, workspace_item)
class _GitPackage(NamedTuple):
path: str
package: _TomlType
workspace: Optional[_TomlType]
workspace: _TomlType | None
@property
def normalized(self) -> _TomlType:
@@ -175,27 +184,26 @@ class _GitPackage(NamedTuple):
return package
_GitPackagesType = Dict[str, _GitPackage]
_GitPackagesType = dict[str, _GitPackage]
async def get_git_repo_packages(git_url: str, commit: str) -> _GitPackagesType:
logging.info('Loading packages from %s', git_url)
logger.info("Loading packages from %s", git_url)
git_repo_dir = fetch_git_repo(git_url, commit)
packages: _GitPackagesType = {}
def get_cargo_toml_packages(root_dir: str, workspace: Optional[_TomlType] = None):
def get_cargo_toml_packages(root_dir: str, workspace: _TomlType | None = None):
assert not os.path.isabs(root_dir) and os.path.isdir(root_dir)
with workdir(root_dir):
if os.path.exists('Cargo.toml'):
cargo_toml = load_toml('Cargo.toml')
workspace = cargo_toml.get('workspace') or workspace
if os.path.exists("Cargo.toml"):
cargo_toml = load_toml("Cargo.toml")
workspace = cargo_toml.get("workspace") or workspace
if 'package' in cargo_toml:
packages[cargo_toml['package']['name']] = _GitPackage(
path=os.path.normpath(root_dir),
package=cargo_toml,
workspace=workspace
if "package" in cargo_toml:
packages[cargo_toml["package"]["name"]] = _GitPackage(
path=os.path.normpath(root_dir), package=cargo_toml, workspace=workspace
)
for child in os.scandir(root_dir):
if child.is_dir():
@@ -203,11 +211,11 @@ async def get_git_repo_packages(git_url: str, commit: str) -> _GitPackagesType:
get_cargo_toml_packages(child.path, workspace)
with workdir(git_repo_dir):
get_cargo_toml_packages('.')
get_cargo_toml_packages(".")
assert packages, f"No packages found in {git_repo_dir}"
logging.debug(
'Packages in %s:\n%s',
logger.debug(
"Packages in %s:\n%s",
git_url,
json.dumps(
{k: v.path for k, v in packages.items()},
@@ -217,99 +225,104 @@ async def get_git_repo_packages(git_url: str, commit: str) -> _GitPackagesType:
return packages
_FlatpakSourceType = Dict[str, Any]
_FlatpakSourceType = dict[str, Any]
async def get_git_repo_sources(
url: str,
commit: str,
tarball: bool = False,
) -> List[_FlatpakSourceType]:
) -> list[_FlatpakSourceType]:
name = git_repo_name(url, commit)
if tarball:
tarball_url = get_git_tarball(url, commit)
git_repo_sources = [{
'type': 'archive',
'archive-type': 'tar-gzip',
'url': tarball_url,
'sha256': await get_remote_sha256(tarball_url),
'dest': f'{GIT_CACHE}/{name}',
}]
git_repo_sources = [
{
"type": "archive",
"archive-type": "tar-gzip",
"url": tarball_url,
"sha256": await get_remote_sha256(tarball_url),
"dest": f"{GIT_CACHE}/{name}",
}
]
else:
git_repo_sources = [{
'type': 'git',
'url': url,
'commit': commit,
'dest': f'{GIT_CACHE}/{name}',
}]
git_repo_sources = [
{
"type": "git",
"url": url,
"commit": commit,
"dest": f"{GIT_CACHE}/{name}",
}
]
return git_repo_sources
_GitRepo = TypedDict('_GitRepo', {'lock': asyncio.Lock, 'commits': Dict[str, _GitPackagesType]})
_GitReposType = Dict[str, _GitRepo]
_VendorEntryType = Dict[str, Dict[str, str]]
_GitRepo = TypedDict("_GitRepo", {"lock": asyncio.Lock, "commits": dict[str, _GitPackagesType]})
_GitReposType = dict[str, _GitRepo]
_VendorEntryType = dict[str, dict[str, str]]
async def get_git_package_sources(
package: _TomlType,
git_repos: _GitReposType,
) -> Tuple[List[_FlatpakSourceType], _VendorEntryType]:
name = package['name']
source = package['source']
) -> tuple[list[_FlatpakSourceType], _VendorEntryType]:
name = package["name"]
source = package["source"]
commit = urlparse(source).fragment
assert commit, 'The commit needs to be indicated in the fragement part'
assert commit, "The commit needs to be indicated in the fragement part"
canonical = canonical_url(source)
repo_url = canonical.geturl()
git_repo = git_repos.setdefault(repo_url, {
'commits': {},
'lock': asyncio.Lock(),
})
async with git_repo['lock']:
if commit not in git_repo['commits']:
git_repo['commits'][commit] = await get_git_repo_packages(repo_url, commit)
git_repo = git_repos.setdefault(
repo_url,
{
"commits": {},
"lock": asyncio.Lock(),
},
)
async with git_repo["lock"]:
if commit not in git_repo["commits"]:
git_repo["commits"][commit] = await get_git_repo_packages(repo_url, commit)
cargo_vendored_entry: _VendorEntryType = {
repo_url: {
'git': repo_url,
'replace-with': VENDORED_SOURCES,
"git": repo_url,
"replace-with": VENDORED_SOURCES,
}
}
rev = parse_qs(urlparse(source).query).get('rev')
tag = parse_qs(urlparse(source).query).get('tag')
branch = parse_qs(urlparse(source).query).get('branch')
rev = parse_qs(urlparse(source).query).get("rev")
tag = parse_qs(urlparse(source).query).get("tag")
branch = parse_qs(urlparse(source).query).get("branch")
if rev:
assert len(rev) == 1
cargo_vendored_entry[repo_url]['rev'] = rev[0]
cargo_vendored_entry[repo_url]["rev"] = rev[0]
elif tag:
assert len(tag) == 1
cargo_vendored_entry[repo_url]['tag'] = tag[0]
cargo_vendored_entry[repo_url]["tag"] = tag[0]
elif branch:
assert len(branch) == 1
cargo_vendored_entry[repo_url]['branch'] = branch[0]
cargo_vendored_entry[repo_url]["branch"] = branch[0]
logging.info("Adding package %s from %s", name, repo_url)
git_pkg = git_repo['commits'][commit][name]
logger.info("Adding package %s from %s", name, repo_url)
git_pkg = git_repo["commits"][commit][name]
pkg_repo_dir = os.path.join(GIT_CACHE, git_repo_name(repo_url, commit), git_pkg.path)
git_sources: List[_FlatpakSourceType] = [
git_sources: list[_FlatpakSourceType] = [
{
'type': 'shell',
'commands': [
f'cp -r --reflink=auto "{pkg_repo_dir}" "{CARGO_CRATES}/{name}"'
],
"type": "shell",
"commands": [f'cp -r --reflink=auto "{pkg_repo_dir}" "{CARGO_CRATES}/{name}"'],
},
{
'type': 'inline',
'contents': toml.dumps(git_pkg.normalized),
'dest': f'{CARGO_CRATES}/{name}', #-{version}',
'dest-filename': 'Cargo.toml',
"type": "inline",
"contents": toml.dumps(git_pkg.normalized),
"dest": f"{CARGO_CRATES}/{name}", # -{version}',
"dest-filename": "Cargo.toml",
},
{
'type': 'inline',
'contents': json.dumps({'package': None, 'files': {}}),
'dest': f'{CARGO_CRATES}/{name}', #-{version}',
'dest-filename': '.cargo-checksum.json',
}
"type": "inline",
"contents": json.dumps({"package": None, "files": {}}),
"dest": f"{CARGO_CRATES}/{name}", # -{version}',
"dest-filename": ".cargo-checksum.json",
},
]
return (git_sources, cargo_vendored_entry)
@@ -319,49 +332,49 @@ async def get_package_sources(
package: _TomlType,
cargo_lock: _TomlType,
git_repos: _GitReposType,
) -> Optional[Tuple[List[_FlatpakSourceType], _VendorEntryType]]:
metadata = cargo_lock.get('metadata')
name = package['name']
version = package['version']
) -> tuple[list[_FlatpakSourceType], _VendorEntryType] | None:
metadata = cargo_lock.get("metadata")
name = package["name"]
version = package["version"]
if 'source' not in package:
logging.debug('%s has no source', name)
if "source" not in package:
logger.debug("%s has no source", name)
return None
source = package['source']
source = package["source"]
if source.startswith('git+'):
if source.startswith("git+"):
return await get_git_package_sources(package, git_repos)
key = f'checksum {name} {version} ({source})'
key = f"checksum {name} {version} ({source})"
if metadata is not None and key in metadata:
checksum = metadata[key]
elif 'checksum' in package:
checksum = package['checksum']
elif "checksum" in package:
checksum = package["checksum"]
else:
logging.warning(f'{name} doesn\'t have checksum')
logger.warning(f"{name} doesn't have checksum")
return None
crate_sources = [
{
'type': 'archive',
'archive-type': 'tar-gzip',
'url': f'{CRATES_IO}/{name}/{name}-{version}.crate',
'sha256': checksum,
'dest': f'{CARGO_CRATES}/{name}-{version}',
"type": "archive",
"archive-type": "tar-gzip",
"url": f"{CRATES_IO}/{name}/{name}-{version}.crate",
"sha256": checksum,
"dest": f"{CARGO_CRATES}/{name}-{version}",
},
{
'type': 'inline',
'contents': json.dumps({'package': checksum, 'files': {}}),
'dest': f'{CARGO_CRATES}/{name}-{version}',
'dest-filename': '.cargo-checksum.json',
"type": "inline",
"contents": json.dumps({"package": checksum, "files": {}}),
"dest": f"{CARGO_CRATES}/{name}-{version}",
"dest-filename": ".cargo-checksum.json",
},
]
return (crate_sources, {'crates-io': {'replace-with': VENDORED_SOURCES}})
return (crate_sources, {"crates-io": {"replace-with": VENDORED_SOURCES}})
async def generate_sources(
cargo_lock: _TomlType,
git_tarballs: bool = False,
) -> List[_FlatpakSourceType]:
) -> list[_FlatpakSourceType]:
# {
# "git-repo-url": {
# "lock": asyncio.Lock(),
@@ -373,13 +386,13 @@ async def generate_sources(
# }
# }
git_repos: _GitReposType = {}
sources: List[_FlatpakSourceType] = []
sources: list[_FlatpakSourceType] = []
package_sources = []
cargo_vendored_sources = {
VENDORED_SOURCES: {'directory': f'{CARGO_CRATES}'},
VENDORED_SOURCES: {"directory": f"{CARGO_CRATES}"},
}
pkg_coros = [get_package_sources(p, cargo_lock, git_repos) for p in cargo_lock['package']]
pkg_coros = [get_package_sources(p, cargo_lock, git_repos) for p in cargo_lock["package"]]
for pkg in await asyncio.gather(*pkg_coros):
if pkg is None:
continue
@@ -388,49 +401,56 @@ async def generate_sources(
package_sources.extend(pkg_sources)
cargo_vendored_sources.update(cargo_vendored_entry)
logging.debug('Adding collected git repos:\n%s', json.dumps(list(git_repos), indent=4))
logger.debug("Adding collected git repos:\n%s", json.dumps(list(git_repos), indent=4))
git_repo_coros = []
for git_url, git_repo in git_repos.items():
for git_commit in git_repo['commits']:
for git_commit in git_repo["commits"]:
git_repo_coros.append(get_git_repo_sources(git_url, git_commit, git_tarballs))
sources.extend(sum(await asyncio.gather(*git_repo_coros), []))
sources.extend(itertools.chain.from_iterable(await asyncio.gather(*git_repo_coros)))
sources.extend(package_sources)
logging.debug('Vendored sources:\n%s', json.dumps(cargo_vendored_sources, indent=4))
sources.append({
'type': 'inline',
'contents': toml.dumps({
'source': cargo_vendored_sources,
}),
'dest': CARGO_HOME,
'dest-filename': 'config'
})
logger.debug("Vendored sources:\n%s", json.dumps(cargo_vendored_sources, indent=4))
sources.append(
{
"type": "inline",
"contents": toml.dumps(
{
"source": cargo_vendored_sources,
}
),
"dest": CARGO_HOME,
"dest-filename": "config",
}
)
return sources
def main():
parser = argparse.ArgumentParser()
parser.add_argument('cargo_lock', help='Path to the Cargo.lock file')
parser.add_argument('-o', '--output', required=False, help='Where to write generated sources')
parser.add_argument('-t', '--git-tarballs', action='store_true', help='Download git repos as tarballs')
parser.add_argument('-d', '--debug', action='store_true')
parser.add_argument("cargo_lock", help="Path to the Cargo.lock file")
parser.add_argument("-o", "--output", required=False, help="Where to write generated sources")
parser.add_argument(
"-t", "--git-tarballs", action="store_true", help="Download git repos as tarballs"
)
parser.add_argument("-d", "--debug", action="store_true")
args = parser.parse_args()
if args.output is not None:
outfile = args.output
else:
outfile = 'generated-sources.json'
outfile = "generated-sources.json"
if args.debug:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
logging.basicConfig(level=loglevel)
generated_sources = asyncio.run(generate_sources(load_toml(args.cargo_lock),
git_tarballs=args.git_tarballs))
with open(outfile, 'w') as out:
generated_sources = asyncio.run(
generate_sources(load_toml(args.cargo_lock), git_tarballs=args.git_tarballs)
)
with open(outfile, "w") as out:
json.dump(generated_sources, out, indent=4, sort_keys=False)
if __name__ == '__main__':
if __name__ == "__main__":
main()
+36 -31
View File
@@ -1,13 +1,13 @@
#!/usr/bin/env python3
import xml.etree.ElementTree as ET
import argparse
import os
import sys
import typing
import datetime
import email.utils
import os
import subprocess
import sys
import typing
import xml.etree.ElementTree as ET
HEADER = """wivrn ({version}-1) {distribution}; urgency=medium
@@ -21,17 +21,17 @@ FOOTER = """
"""
def write_entry(
out: typing.TextIO,
version: str,
data: list[str],
author: str,
distribution: str,
date: datetime.datetime
) -> None:
date: datetime.datetime,
) -> None:
out.write(HEADER.format(version=version, distribution=distribution))
for item in data:
out.write(ITEM.format(text=item))
out.writelines(ITEM.format(text=item) for item in data)
out.write(FOOTER.format(author=author, date=email.utils.format_datetime(date)))
@@ -45,36 +45,41 @@ def main():
args = parser.parse_args()
if not args.metainfo:
args.metainfo = open(os.path.join(os.path.dirname(__file__), "../dashboard/io.github.wivrn.wivrn.metainfo.xml"))
tree = ET.parse(args.metainfo)
if args.metainfo:
tree = ET.parse(args.metainfo)
else:
default_metainfo = os.path.join(
os.path.dirname(__file__), "../dashboard/io.github.wivrn.wivrn.metainfo.xml"
)
with open(default_metainfo) as metainfo:
tree = ET.parse(metainfo)
if args.nightly:
version = subprocess.check_output(["git", "describe", "--tags", "--always"]).decode().strip()
if version[0] == 'v':
version = version[1:] # remove the initial "v"
version = (
subprocess.check_output(["git", "describe", "--tags", "--always"]).decode().strip()
)
if version[0] == "v":
version = version[1:] # remove the initial "v"
else:
version = '0-' + version
version = "0-" + version
write_entry(
out=args.out,
version=version,
data=["Nightly build"],
author=args.author,
distribution=args.distribution,
date=datetime.datetime.now()
)
out=args.out,
version=version,
data=["Nightly build"],
author=args.author,
distribution=args.distribution,
date=datetime.datetime.now().astimezone(),
)
for release in tree.find("releases").findall("release"):
write_entry(
out=args.out,
version=release.get("version"),
data=[item.text for item in release.findall("description/ul/li")],
author=args.author,
distribution=args.distribution,
date=datetime.datetime.fromisoformat(release.get("date"))
)
out=args.out,
version=release.get("version"),
data=[item.text for item in release.findall("description/ul/li")],
author=args.author,
distribution=args.distribution,
date=datetime.datetime.fromisoformat(release.get("date")),
)
if __name__ == "__main__":
+22 -21
View File
@@ -20,9 +20,7 @@ class CMakeFile:
with open(filename) as f:
data = f.read()
for match in re.findall("FetchContent_Declare\\(([^)]*)\\)", data):
words = [word for word
in match.replace("\n", " ").split(" ")
if word]
words = [word for word in match.replace("\n", " ").split(" ") if word]
self.entries[words[0]] = words[1:]
def get(self, target: str, key: str) -> str:
@@ -36,7 +34,9 @@ if __name__ == "__main__":
parser.add_argument("--git")
parser.add_argument("--gitlocal", action="store_true")
parser.add_argument("--dir", action="store_true")
parser.add_argument("--out", help="directory where to write manifest and additional files", default=".")
parser.add_argument(
"--out", help="directory where to write manifest and additional files", default="."
)
args = parser.parse_args()
if not (args.git or args.dir):
@@ -52,34 +52,35 @@ if __name__ == "__main__":
xrizer_repo = "https://github.com/Supreeeme/xrizer"
xrizer_commit = re.findall(f"url: {xrizer_repo}.git[ \n]*commit: ([a-f0-9]*)", template)[0]
lock = urlopen(f"{xrizer_repo}/raw/{xrizer_commit}/Cargo.lock")
gen_src = asyncio.run(flatpakcargogenerator.generate_sources(
toml.loads(lock.read().decode())))
gen_src = asyncio.run(flatpakcargogenerator.generate_sources(toml.loads(lock.read().decode())))
with open(os.path.join(args.out, "xrizer-gen-src.json"), "w") as f:
json.dump(gen_src, f, indent=4, sort_keys=False)
monado_commit = open(os.path.join(root, "monado-rev")).read()
with open(os.path.join(root, "monado-rev")) as f:
monado_commit = f.read()
boost_url = cmake.get("boost", "URL")
boost_sha256 = cmake.get("boost", "URL_HASH").split("=")[-1]
try:
git_tag = subprocess.check_output(
["git", "describe", "--exact-match", "--tags"],
cwd=root,
stderr=subprocess.DEVNULL
).decode().strip()
git_tag = (
subprocess.check_output(
["git", "describe", "--exact-match", "--tags"], cwd=root, stderr=subprocess.DEVNULL
)
.decode()
.strip()
)
git_commit = ""
git_desc = ""
except subprocess.CalledProcessError:
git_tag = ""
git_commit = subprocess.check_output(
["git", "rev-parse", "HEAD"],
cwd=root
).decode().strip()
git_desc = subprocess.check_output(
["git", "describe", "--tags", "--always"],
cwd=root
).decode().strip()
git_commit = (
subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root).decode().strip()
)
git_desc = (
subprocess.check_output(["git", "describe", "--tags", "--always"], cwd=root)
.decode()
.strip()
)
if args.git or args.gitlocal:
template = template.replace("WIVRN_SRC1", "type: git")
View File
File mode changed.
+16 -7
View File
@@ -18,6 +18,7 @@ import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path
@@ -91,7 +92,7 @@ def download_tracebox(url, dest):
else:
print(f"\r {downloaded / 1e6:.1f} MB", end="", flush=True)
print()
except Exception as e:
except (OSError, urllib.error.URLError) as e:
Path(dest).unlink(missing_ok=True)
sys.exit(f"wivrn_capture: download failed: {e}")
@@ -106,13 +107,15 @@ def ensure_tracebox(path):
def pgrep(name):
"""Return list of matching PIDs (as strings), or []."""
r = subprocess.run(["pgrep", "-x", name], capture_output=True, text=True)
r = subprocess.run(["pgrep", "-x", name], capture_output=True, text=True, check=False)
return r.stdout.split() if r.returncode == 0 else []
def socket_listening(sock_path):
"""Return True if a Unix socket at sock_path is bound and listening."""
r = subprocess.run(["ss", "-lxH", "src", str(sock_path)], capture_output=True, text=True)
r = subprocess.run(
["ss", "-lxH", "src", str(sock_path)], capture_output=True, text=True, check=False
)
return bool(r.stdout.strip())
@@ -121,7 +124,11 @@ def wait_for_producer(tracebox, env, timeout_s=20):
deadline = time.time() + timeout_s
while time.time() < deadline:
r = subprocess.run(
[str(tracebox), "perfetto", "--query"], env=env, capture_output=True, text=True
[str(tracebox), "perfetto", "--query"],
env=env,
capture_output=True,
text=True,
check=False,
)
if "track_event" in r.stdout:
return True
@@ -193,7 +200,7 @@ def main():
if args.output:
out = Path(args.output)
else:
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
ts = datetime.now().astimezone().strftime("%Y%m%d-%H%M%S")
out = Path(f"wivrn-{ts}.pftrace")
if not cfg.is_file():
@@ -234,9 +241,11 @@ def main():
def cleanup():
run_cfg.unlink(missing_ok=True)
if started_probes:
subprocess.run([*sudo, "pkill", "-x", "traced_probes"], capture_output=True)
subprocess.run(
[*sudo, "pkill", "-x", "traced_probes"], capture_output=True, check=False
)
if started_traced:
subprocess.run([*sudo, "pkill", "-x", "traced"], capture_output=True)
subprocess.run([*sudo, "pkill", "-x", "traced"], capture_output=True, check=False)
producer_sock.unlink(missing_ok=True)
consumer_sock.unlink(missing_ok=True)
+18 -7
View File
@@ -1,15 +1,17 @@
#!/usr/bin/env python3
import pandas
import csv
import typing
import pandas
class Frame:
def __init__(self, num):
self.num = num
self.events = dict()
self.streams = dict()
self.flags = dict()
self.events = {}
self.streams = {}
self.flags = {}
self.len = 1
def set(self, event, timestamp, stream):
@@ -17,7 +19,7 @@ class Frame:
self.events[event] = timestamp
else:
if stream not in self.streams:
self.streams[stream] = dict()
self.streams[stream] = {}
if event in ("blit", "display"):
if event not in self.streams[stream]:
self.streams[stream][event] = [timestamp]
@@ -31,6 +33,7 @@ class Frame:
def flag(self, stream, flag):
self.flags[stream if stream != 255 else None] = flag
def read(file: typing.TextIO) -> pandas.DataFrame:
timings = csv.reader(file)
@@ -38,7 +41,6 @@ def read(file: typing.TextIO) -> pandas.DataFrame:
frames = []
streams = 0
for event, frame, timestamp, stream, *extra in timings:
if event == "event":
continue
@@ -68,7 +70,16 @@ def read(file: typing.TextIO) -> pandas.DataFrame:
frames[frame].flag(stream, x)
common_cols = ["wake_up", "begin", "submit"]
stream_cols = ["encode_begin", "encode_end", "send_begin", "send_end", "receive_begin", "receive_end", "decode_begin", "decode_end"]
stream_cols = [
"encode_begin",
"encode_end",
"send_begin",
"send_end",
"receive_begin",
"receive_end",
"decode_begin",
"decode_end",
]
display_cols = ["blit", "display"]
columns = ["frame"] + common_cols + stream_cols + display_cols