WIP: Rust: Add support to install/update crates.io wraps

mesonbuild/msubprojects.py

@@ -12,11 +12,13 @@
 import typing as T
 import tarfile
 import zipfile
+import urllib.parse
 
 from . import mlog
 from .mesonlib import quiet_git, GitException, Popen_safe, MesonException, windows_proof_rmtree
 from .wrap.wrap import (Resolver, WrapException, ALL_TYPES, PackageDefinition,
-                        parse_patch_url, update_wrap_file, get_releases)
+                        parse_patch_url, update_wrap_file, get_releases,
+                        get_crates_io_info, update_crates_io_wrap_file)
 
 if T.TYPE_CHECKING:
     from typing_extensions import Protocol
@@ -135,44 +137,87 @@ def run(self) -> bool:
         return result
 
     @staticmethod
-    def pre_update_wrapdb(options: 'UpdateWrapDBArguments') -> None:
+    def pre_fetch_wrapdb(options: 'UpdateWrapDBArguments') -> None:
         options.releases = get_releases(options.allow_insecure)
 
-    def update_wrapdb(self) -> bool:
-        self.log(f'Checking latest WrapDB version for {self.wrap.name}...')
+    def _get_current_version(self) -> T.Tuple[T.Optional[str], T.Optional[str]]:
+        try:
+            source_url = self.wrap.get('source_url')
+            u = urllib.parse.urlparse(source_url)
+            arr = u.path.strip('/').split('/')
+            if u.netloc == 'crates.io' and arr[1] == 'v1':
+                return arr[4], arr[3]
+        except WrapException:
+            pass
+        try:
+            return self.wrap.get('wrapdb_version'), None
+        except WrapException:
+            pass
+        # Fallback to parsing the patch URL to determine current version.
+        # This won't work for projects that have upstream Meson support.
+        try:
+            patch_url = self.wrap.get('patch_url')
+            branch, revision = parse_patch_url(patch_url)
+            return f'{branch}-{revision}', None
+        except WrapException:
+            return None, None
+
+    def wrap_update(self) -> bool:
+        self.log(f'Checking latest version for {self.wrap.name}...')
         options = T.cast('UpdateWrapDBArguments', self.options)
 
-        # Check if this wrap is in WrapDB
-        info = options.releases.get(self.wrap.name)
-        if not info:
-            self.log('  -> Wrap not found in wrapdb')
-            return True
+        # Check if this wrap is in WrapDB or crates.io
+        version, crates_io_name = self._get_current_version()
+        if crates_io_name:
+            latest_version, _, _ = get_crates_io_info(crates_io_name, options.allow_insecure)
+        else:
+            info = options.releases.get(self.wrap.name)
+            if not info:
+                self.log('  -> Wrap not found in wrapdb')
+                return True
+            latest_version = info['versions'][0]
 
-        # Determine current version
-        try:
-            wrapdb_version = self.wrap.get('wrapdb_version')
-            branch, revision = wrapdb_version.split('-', 1)
-        except WrapException:
-            # Fallback to parsing the patch URL to determine current version.
-            # This won't work for projects that have upstream Meson support.
-            try:
-                patch_url = self.wrap.get('patch_url')
-                branch, revision = parse_patch_url(patch_url)
-            except WrapException:
-                if not options.force:
-                    self.log('  ->', mlog.red('Could not determine current version, use --force to update any way'))
-                    return False
-                branch = revision = None
+        if version is None and not options.force:
+            self.log('  ->', mlog.red('Could not determine current version, use --force to update any way'))
+            return False
 
         # Download latest wrap if version differs
-        latest_version = info['versions'][0]
-        new_branch, new_revision = latest_version.rsplit('-', 1)
-        if new_branch != branch or new_revision != revision:
+        if version != latest_version:
             filename = self.wrap.filename if self.wrap.has_wrap else f'{self.wrap.filename}.wrap'
-            update_wrap_file(filename, self.wrap.name,
-                             new_branch, new_revision,
-                             options.allow_insecure)
-            self.log('  -> New version downloaded:', mlog.blue(latest_version))
+            if crates_io_name:
+                update_crates_io_wrap_file(filename, self.wrap.name, crates_io_name, options.allow_insecure)
+            else:
+                new_branch, new_revision = latest_version.rsplit('-', 1)
+                update_wrap_file(filename, self.wrap.name,
+                                 new_branch, new_revision,
+                                 options.allow_insecure)
+            self.log('  -> New version downloaded:', mlog.blue(version), '→', mlog.blue(latest_version))
+        else:
+            self.log('  -> Already at latest version:', mlog.blue(latest_version))
+
+        return True
+
+    def wrap_status(self) -> bool:
+        self.log(f'Checking latest version for {self.wrap.name}...')
+        options = T.cast('UpdateWrapDBArguments', self.options)
+
+        # Check if this wrap is in WrapDB or crates.io
+        version, crates_io_name = self._get_current_version()
+        if crates_io_name:
+            latest_version, _, _ = get_crates_io_info(crates_io_name, options.allow_insecure)
+        else:
+            info = options.releases.get(self.wrap.name)
+            if not info:
+                self.log('  -> Wrap not found in wrapdb')
+                return True
+            latest_version = info['versions'][0]
+
+        if version is None:
+            self.log('  ->', mlog.red('Could not determine current version'))
+            return False
+
+        if version != latest_version:
+            self.log('  -> New version available:', mlog.blue(version), '→', mlog.blue(latest_version))
         else:
             self.log('  -> Already at latest version:', mlog.blue(latest_version))
 
@@ -626,8 +671,16 @@ def add_wrap_update_parser(subparsers: 'SubParsers') -> argparse.ArgumentParser:
                    help='Update wraps that does not seems to come from WrapDB')
     add_common_arguments(p)
     add_subprojects_argument(p)
-    p.set_defaults(subprojects_func=Runner.update_wrapdb)
-    p.set_defaults(pre_func=Runner.pre_update_wrapdb)
+    p.set_defaults(subprojects_func=Runner.wrap_update)
+    p.set_defaults(pre_func=Runner.pre_fetch_wrapdb)
+    return p
+
+def add_wrap_status_parser(subparsers: 'SubParsers') -> argparse.ArgumentParser:
+    p = subparsers.add_parser('status', help='Show installed and available versions of your projects')
+    add_common_arguments(p)
+    add_subprojects_argument(p)
+    p.set_defaults(subprojects_func=Runner.wrap_status)
+    p.set_defaults(pre_func=Runner.pre_fetch_wrapdb)
     return p
 
 def add_arguments(parser: argparse.ArgumentParser) -> None:

mesonbuild/wrap/wrap.py

@@ -72,18 +72,18 @@ def whitelist_wrapdb(urlstr: str) -> urllib.parse.ParseResult:
         raise WrapException(f'WrapDB did not have expected SSL https url, instead got {urlstr}')
     return url
 
-def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool = False) -> 'http.client.HTTPResponse':
+def open_url(urlstring: str, allow_insecure: bool = False, have_opt: bool = False) -> 'http.client.HTTPResponse':
     if have_opt:
         insecure_msg = '\n\n    To allow connecting anyway, pass `--allow-insecure`.'
     else:
         insecure_msg = ''
 
-    url = whitelist_wrapdb(urlstring)
+    url = urllib.parse.urlparse(urlstring)
     if has_ssl:
         try:
             return T.cast('http.client.HTTPResponse', urllib.request.urlopen(urllib.parse.urlunparse(url), timeout=REQ_TIMEOUT))
         except urllib.error.URLError as excp:
-            msg = f'WrapDB connection failed to {urlstring} with error {excp}.'
+            msg = f'Connection failed to {urlstring} with error {excp}.'
             if isinstance(excp.reason, ssl.SSLCertVerificationError):
                 if allow_insecure:
                     mlog.warning(f'{msg}\n\n    Proceeding without authentication.')
@@ -92,17 +92,21 @@ def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool
             else:
                 raise WrapException(msg)
     elif not allow_insecure:
-        raise WrapException(f'SSL module not available in {sys.executable}: Cannot contact the WrapDB.{insecure_msg}')
+        raise WrapException(f'SSL module not available in {sys.executable}: Cannot contact the server.{insecure_msg}')
     else:
         # following code is only for those without Python SSL
-        mlog.warning(f'SSL module not available in {sys.executable}: WrapDB traffic not authenticated.', once=True)
+        mlog.warning(f'SSL module not available in {sys.executable}: traffic not authenticated.', once=True)
 
     # If we got this far, allow_insecure was manually passed
     nossl_url = url._replace(scheme='http')
     try:
         return T.cast('http.client.HTTPResponse', urllib.request.urlopen(urllib.parse.urlunparse(nossl_url), timeout=REQ_TIMEOUT))
     except urllib.error.URLError as excp:
-        raise WrapException(f'WrapDB connection failed to {urlstring} with error {excp}')
+        raise WrapException(f'Connection failed to {urlstring} with error {excp}')
+
+def open_wrapdburl(urlstring: str, allow_insecure: bool = False, have_opt: bool = False) -> 'http.client.HTTPResponse':
+    whitelist_wrapdb(urlstring)
+    return open_url(urlstring, allow_insecure, have_opt)
 
 def get_releases_data(allow_insecure: bool) -> bytes:
     url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True)
@@ -135,6 +139,32 @@ def parse_patch_url(patch_url: str) -> T.Tuple[str, str]:
     else:
         raise WrapException(f'Invalid wrapdb URL {patch_url}')
 
+CRATES_IO_WRAP_TMPL = '''\
+[wrap-file]
+directory = {0}-{1}
+source_url = https://crates.io{2}
+source_filename = {0}-{1}.tar.gz
+source_hash = {3}
+
+[provide]
+dependency_names = {4}
+'''
+
+def get_crates_io_info(name: str, allow_insecure: bool) -> T.Tuple[str, str, str]:
+    url = open_url(f'https://crates.io/api/v1/crates/{name}/versions', allow_insecure)
+    try:
+        data = json.loads(url.read().decode())
+        info = data['versions'][0]
+        return info['num'], info['dl_path'], info['checksum'],
+    except:
+        raise WrapException('crate.io reply not in expected format')
+
+def update_crates_io_wrap_file(wrapfile: str, name: str, crates_io_name: str, allow_insecure: bool) -> str:
+    version, dl_path, checksum = get_crates_io_info(crates_io_name, allow_insecure)
+    with open(wrapfile, 'w', encoding='utf-8') as f:
+        f.write(CRATES_IO_WRAP_TMPL.format(crates_io_name, version, dl_path, checksum, name))
+    return version
+
 class WrapException(MesonException):
     pass
 

mesonbuild/wrap/wraptool.py

@@ -19,8 +19,7 @@
 import typing as T
 
 from glob import glob
-from .wrap import (open_wrapdburl, WrapException, get_releases, get_releases_data,
-                   update_wrap_file, parse_patch_url)
+from .wrap import open_wrapdburl, WrapException, get_releases, get_releases_data, update_crates_io_wrap_file
 from pathlib import Path
 
 from .. import mesonlib, msubprojects
@@ -46,6 +45,8 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
     p = subparsers.add_parser('install', help='install the specified project')
     p.add_argument('--allow-insecure', default=False, action='store_true',
                    help='Allow insecure server connections.')
+    p.add_argument('--crates-io', default=False, action='store_true',
+                   help='Use Rust projects from crates.io.')
     p.add_argument('name')
     p.set_defaults(wrap_func=install)
 
@@ -58,10 +59,8 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
     p.add_argument('name')
     p.set_defaults(wrap_func=info)
 
-    p = subparsers.add_parser('status', help='show installed and available versions of your projects')
-    p.add_argument('--allow-insecure', default=False, action='store_true',
-                   help='Allow insecure server connections.')
-    p.set_defaults(wrap_func=status)
+    p = msubprojects.add_wrap_status_parser(subparsers)
+    p.set_defaults(wrap_func=msubprojects.run)
 
     p = subparsers.add_parser('promote', help='bring a subsubproject up to the master project')
     p.add_argument('project_path')
@@ -99,63 +98,24 @@ def get_latest_version(name: str, allow_insecure: bool) -> T.Tuple[str, str]:
 
 def install(options: 'argparse.Namespace') -> None:
     name = options.name
+    if options.crates_io:
+        name = name if name.endswith('-rs') else f'{name}-rs'
     if not os.path.isdir('subprojects'):
         raise SystemExit('Subprojects dir not found. Run this script in your source root directory.')
     if os.path.isdir(os.path.join('subprojects', name)):
         raise SystemExit('Subproject directory for this project already exists.')
     wrapfile = os.path.join('subprojects', name + '.wrap')
     if os.path.exists(wrapfile):
         raise SystemExit('Wrap file already exists.')
-    (version, revision) = get_latest_version(name, options.allow_insecure)
-    url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{version}-{revision}/{name}.wrap', options.allow_insecure, True)
-    with open(wrapfile, 'wb') as f:
-        f.write(url.read())
-    print(f'Installed {name} version {version} revision {revision}')
-
-def get_current_version(wrapfile: str) -> T.Tuple[str, str, str, str, T.Optional[str]]:
-    cp = configparser.ConfigParser(interpolation=None)
-    cp.read(wrapfile)
-    try:
-        wrap_data = cp['wrap-file']
-    except KeyError:
-        raise WrapException('Not a wrap-file, cannot have come from the wrapdb')
-    try:
-        patch_url = wrap_data['patch_url']
-    except KeyError:
-        # We assume a wrap without a patch_url is probably just an pointer to upstream's
-        # build files. The version should be in the tarball filename, even if it isn't
-        # purely guaranteed. The wrapdb revision should be 1 because it just needs uploading once.
-        branch = mesonlib.search_version(wrap_data['source_filename'])
-        revision, patch_filename = '1', None
+    if options.crates_io:
+        version = update_crates_io_wrap_file(wrapfile, name, options.name, options.allow_insecure)
+        print(f'Installed {name} version {version}')
     else:
-        branch, revision = parse_patch_url(patch_url)
-        patch_filename = wrap_data['patch_filename']
-    return branch, revision, wrap_data['directory'], wrap_data['source_filename'], patch_filename
-
-def update(options: 'argparse.Namespace') -> None:
-    name = options.name
-    if not os.path.isdir('subprojects'):
-        raise SystemExit('Subprojects dir not found. Run this command in your source root directory.')
-    wrapfile = os.path.join('subprojects', name + '.wrap')
-    if not os.path.exists(wrapfile):
-        raise SystemExit('Project ' + name + ' is not in use.')
-    (branch, revision, subdir, src_file, patch_file) = get_current_version(wrapfile)
-    (new_branch, new_revision) = get_latest_version(name, options.allow_insecure)
-    if new_branch == branch and new_revision == revision:
-        print('Project ' + name + ' is already up to date.')
-        raise SystemExit
-    update_wrap_file(wrapfile, name, new_branch, new_revision, options.allow_insecure)
-    shutil.rmtree(os.path.join('subprojects', subdir), ignore_errors=True)
-    try:
-        os.unlink(os.path.join('subprojects/packagecache', src_file))
-    except FileNotFoundError:
-        pass
-    if patch_file is not None:
-        try:
-            os.unlink(os.path.join('subprojects/packagecache', patch_file))
-        except FileNotFoundError:
-            pass
-    print(f'Updated {name} version {new_branch} revision {new_revision}')
+        (version, revision) = get_latest_version(name, options.allow_insecure)
+        url = open_wrapdburl(f'https://wrapdb.mesonbuild.com/v2/{name}_{version}-{revision}/{name}.wrap', options.allow_insecure, True)
+        with open(wrapfile, 'wb') as f:
+            f.write(url.read())
+        print(f'Installed {name} version {version} revision {revision}')
 
 def info(options: 'argparse.Namespace') -> None:
     name = options.name
@@ -201,25 +161,6 @@ def promote(options: 'argparse.Namespace') -> None:
         raise SystemExit(1)
     do_promotion(matches[0], spdir_name)
 
-def status(options: 'argparse.Namespace') -> None:
-    print('Subproject status')
-    for w in glob('subprojects/*.wrap'):
-        name = os.path.basename(w)[:-5]
-        try:
-            (latest_branch, latest_revision) = get_latest_version(name, options.allow_insecure)
-        except Exception:
-            print('', name, 'not available in wrapdb.', file=sys.stderr)
-            continue
-        try:
-            (current_branch, current_revision, _, _, _) = get_current_version(w)
-        except Exception:
-            print('', name, 'Wrap file not from wrapdb.', file=sys.stderr)
-            continue
-        if current_branch == latest_branch and current_revision == latest_revision:
-            print('', name, f'up to date. Branch {current_branch}, revision {current_revision}.')
-        else:
-            print('', name, f'not up to date. Have {current_branch} {current_revision}, but {latest_branch} {latest_revision} is available.')
-
 def update_db(options: 'argparse.Namespace') -> None:
     data = get_releases_data(options.allow_insecure)
     Path('subprojects').mkdir(exist_ok=True)