get_package.py

#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright (c) 2018 The ungoogled-chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Simple package script generator.
"""

import argparse
import os
import re
import shutil
import string
import subprocess
from pathlib import Path

from buildkit.common import (ENCODING, BuildkitAbort, get_logger, validate_and_get_ini,
                             get_chromium_version, get_release_revision)
from buildkit.third_party import schema

# Constants

_ROOT_DIR = Path(__file__).resolve().parent
_PACKAGING_ROOT = _ROOT_DIR / 'packaging'
_PKGMETA = _PACKAGING_ROOT / 'pkgmeta.ini'
_TEMPLATE_SUFFIX = '.ungoogin'
_PKGMETA_SCHEMA = schema.Schema({
    schema.Optional(schema.And(str, len)): {
        schema.Optional('depends'): schema.And(str, len),
        schema.Optional('buildkit_copy'): schema.And(str, len),
    }
})

# Classes


class _BuildFileStringTemplate(string.Template):
    """
    Custom string substitution class

    Inspired by
    http://stackoverflow.com/questions/12768107/string-substitutions-using-templates-in-python
    """

    pattern = r"""
    {delim}(?:
      (?P<escaped>{delim}) |
      _(?P<named>{id})      |
      {{(?P<braced>{id})}}   |
      (?P<invalid>{delim}((?!_)|(?!{{)))
    )
    """.format(
        delim=re.escape("$ungoog"), id=string.Template.idpattern)


# Methods


def _process_templates(template_files, build_file_subs):
    """
    Recursively substitute '$ungoog' strings in template_files and
        remove the suffix

    template_files is an iterable of pathlib.Path
    """
    for old_path in template_files:
        new_path = old_path.with_name(old_path.stem)
        old_path.replace(new_path)
        with new_path.open('r+', encoding=ENCODING) as new_file:
            content = _BuildFileStringTemplate(new_file.read()).substitute(**build_file_subs)
            new_file.seek(0)
            new_file.write(content)
            new_file.truncate()


def _get_current_commit_or_tag():
    """
    Returns a string of the current commit hash, or the tag name based
    on version.ini if the script is not in a git repo.

    It assumes "git" is in PATH, and that buildkit is run within a git repository.

    Raises BuildkitAbort if invoking git fails.
    """
    # Use presence of .git directory to determine if using git or not
    # since git will be aggressive in finding a git repository otherwise
    if not (Path(__file__).parent / '.git').exists(): #pylint: disable=no-member
        # Probably not in a git checkout; extrapolate tag name based on
        # version.ini
        return '{}-{}'.format(get_chromium_version(), get_release_revision())
    result = subprocess.run(['git', 'rev-parse', '--verify', 'HEAD'],
                            stdout=subprocess.PIPE,
                            universal_newlines=True,
                            cwd=str(Path(__file__).resolve().parent))
    if result.returncode:
        get_logger().error('Unexpected return code %s', result.returncode)
        get_logger().error('Command output: %s', result.stdout)
        raise BuildkitAbort()
    return result.stdout.strip('\n')


def _get_package_dir_list(package, pkgmeta):
    """
    Returns a list of pathlib.Path to packaging directories to be copied,
        ordered by dependencies first.

    Raises FileNotFoundError if a package directory cannot be found.
    """
    package_list = list()
    current_name = package
    while current_name:
        package_list.append(_PACKAGING_ROOT / current_name)
        if not package_list[-1].exists(): #pylint: disable=no-member
            raise FileNotFoundError(package_list[-1])
        if current_name in pkgmeta and 'depends' in pkgmeta[current_name]:
            current_name = pkgmeta[current_name]['depends']
        else:
            break
    package_list.reverse()
    return package_list


def _get_package_files(package_dir_list):
    """Yields tuples of relative and full package file paths"""
    resolved_files = dict()
    for package_dir in package_dir_list:
        for file_path in package_dir.rglob('*'):
            relative_path = file_path.relative_to(package_dir)
            resolved_files[relative_path] = file_path
    yield from sorted(resolved_files.items())


def _get_buildkit_copy(package, pkgmeta):
    """
    Returns a pathlib.Path relative to the output directory to copy buildkit and bundles to,
        otherwise returns None if buildkit does not need to be copied.
    """
    while package:
        if package in pkgmeta:
            if 'buildkit_copy' in pkgmeta[package]:
                return Path(pkgmeta[package]['buildkit_copy'])
            if 'depends' in pkgmeta[package]:
                package = pkgmeta[package]['depends']
            else:
                break
        else:
            break
    return None

def _escape_string(value):
    return value.replace('"', '\\"')

def _get_spec_format_patch_list(config_bundles_dir, patch_stage_dir):
    """ Returns information about patch files that are fed into the OpenSUSE RPM SPEC file """
    patch_string = ''
    patch_list = []
    with open(config_bundles_dir + '/common/patch_order.list') as series_file:
        patch_list = series_file.readlines()
    with open(config_bundles_dir + '/opensuse/patch_order.list') as series_file:
        temp = series_file.readlines()
        patch_list.extend(temp)
    i = 1
    patch_list_len = len(patch_list)
    for patch_file in patch_list:
        if len(patch_file) < 2:
            patch_list_len = patch_list_len - 1
            continue
        # Copy patch into staging dir
        shutil.copy2('patches/{0}'.format(patch_file[0:len(patch_file)-1]), patch_stage_dir)
        # Add patch info to SPEC file
        last_slash_pos = patch_file.rfind('/')
        patch_file = patch_file[last_slash_pos + 1:]
        patch_string += 'Patch{0}:         {1}'.format(i, patch_file)
        i += 1
    return {'patchString': patch_string, 'numPatches': patch_list_len}

def _get_patch_apply_spec_cmd(num_patches):
    patch_apply_string = ''
    for i in range(1, num_patches + 1):
        patch_apply_string += '%patch{0} -p1\n'.format(i)
    return patch_apply_string

def _get_gn_flag_map(config_bundles_dir):
    gn_flags = dict() 
    def _parse_gn_file(gn_file, gn_flags):
        for line in gn_file:
            delim_idx = line.find('=')
            if delim_idx < 0:
                continue
            key = line[0:delim_idx]
            val = line[delim_idx + 1:len(line) - 1]
            gn_flags[key] = val
        return gn_flags

    with open(config_bundles_dir + '/common/gn_flags.map') as gn_file:
        gn_flags = _parse_gn_file(gn_file, gn_flags)
    with open(config_bundles_dir + '/opensuse/gn_flags.map') as gn_file:
        gn_flags = _parse_gn_file(gn_file, gn_flags)
    return gn_flags

def _get_parsed_gn_flags(gn_flags):
    def _shell_line_generator(gn_flags):
        for key, value in gn_flags.items():
            yield "myconf_gn+=\" " + _escape_string(key) + "=" + _escape_string(value) + "\""
    return os.linesep.join(_shell_line_generator(gn_flags))

def _validate_and_get_args(parser):
    """
    Helper for main()

    Validates and returns arguments
    """
    args = parser.parse_args()
    if not args.destination.parent.exists():
        parser.error('Destination parent directory "{}" does not exist'.format(
            args.destination.parent))
    if not _PACKAGING_ROOT.exists(): #pylint: disable=no-member
        parser.error('Cannot find "packaging" directory next to this script')
    packaging_dir = _PACKAGING_ROOT / args.name
    if not packaging_dir.exists():
        parser.error('Packaging "{}" does not exist'.format(args.name))
    if not _PKGMETA.exists(): #pylint: disable=no-member
        parser.error('Cannot find pkgmeta.ini in packaging directory')
    return args


def _copy_buildkit_files(args, pkgmeta, files, dirs):
    """Helper for main()"""
    buildkit_copy_relative = _get_buildkit_copy(args.name, pkgmeta)
    if buildkit_copy_relative is None:
        return
    for file_name in files:
        if not (args.destination / buildkit_copy_relative).exists():
            (args.destination / buildkit_copy_relative).mkdir()
        shutil.copy(
            str(_ROOT_DIR / file_name), str(args.destination / buildkit_copy_relative / file_name))
    for dir_name in dirs:
        if (args.destination / buildkit_copy_relative / dir_name).exists():
            shutil.rmtree(str(args.destination / buildkit_copy_relative / dir_name))
        shutil.copytree(
            str(_ROOT_DIR / dir_name), str(args.destination / buildkit_copy_relative / dir_name))


def main():
    """CLI Entrypoint"""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('name', help='Name of packaging to generate')
    parser.add_argument('destination', type=Path, help='Directory to store packaging files')
    args = _validate_and_get_args(parser)

    if not args.destination.exists():
        args.destination.mkdir()

    # Copy packaging files to destination
    pkgmeta = validate_and_get_ini(_PKGMETA, _PKGMETA_SCHEMA)
    template_files = set()
    for relative_path, actual_path in _get_package_files(_get_package_dir_list(args.name, pkgmeta)):
        target_path = args.destination / relative_path
        if actual_path.is_dir():
            if not target_path.exists():
                target_path.mkdir()
            shutil.copymode(str(actual_path), str(target_path))
        else:
            shutil.copy(str(actual_path), str(target_path))
            if target_path.suffix.lower() == _TEMPLATE_SUFFIX:
                template_files.add(target_path)

    patch_stage_dir = str(args.destination / 'patches')
    os.mkdir(patch_stage_dir)
    patch_info = _get_spec_format_patch_list('config_bundles', patch_stage_dir) 
    gn_map = _get_gn_flag_map('config_bundles')

    # Substitute template files
    packaging_subs = dict(
        chromium_version=get_chromium_version(),
        release_revision=get_release_revision(),
        numbered_patch_list=patch_info['patchString'],
        apply_patches_cmd=_get_patch_apply_spec_cmd(patch_info['numPatches']),
        gn_flags=_get_parsed_gn_flags(gn_map),
        current_commit_or_tag=_get_current_commit_or_tag()
    )
    _process_templates(template_files, packaging_subs)

    # Copy buildkit and config files, if necessary
    _copy_buildkit_files(
        args,
        pkgmeta,
        files=('version.ini', 'run_buildkit_cli.py'),
        dirs=('buildkit', 'config_bundles', 'patches'))


if __name__ == '__main__':
    main()