From c11a237ca1fc89fc0bd075331db732445032a0c3 Mon Sep 17 00:00:00 2001 From: Nick Virag Date: Mon, 17 Aug 2020 12:53:37 -0400 Subject: [PATCH] Fix: Amazon Linux builds --- README.md | 65 +++++++++++++++++++++++++++++++++ cerbero-python3.sh | 26 +++++++++++++ cerbero/packages/disttarball.py | 2 +- cerbero/packages/rpm.py | 23 +++++++++--- 4 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 cerbero-python3.sh diff --git a/README.md b/README.md index 3bc8408c1..8e2b1faff 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,68 @@ +# Waymark Readme + +Cerbero is the build system for gstreamer. Cerbero requires some fixes to run properly on Amazon Linux, which are included in this repository. + +Here are the modified build instructions for Amazon Linux. The original Cerbero readme is further below. + +## Requirements + + - Amazon Linux 2 EC2 instance or [virtual machine](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/amazon-linux-2-virtual-machine.html) + - 12gb available disk space. WARNING: The default EBS volume size for many EC2 instance types (8gb) is too small. + - 4gb ram + +## Install dependencies +``` +# Ensure system is up-to-date +sudo yum update +sudo reboot + +# Install python 3, git, and alsa-lib-devel. +# alsa-lib-devel is a gstreamer dependency that cerbero will fail to install, +# likely because the alsa dev package has many different names on different +# platforms. cerbero generally succeeds at automatically installing other +# dependencies. +sudo yum install python3 git alsa-lib-devel +``` + +## Create and activate python 3 virtual environment +``` +python3 -m venv cerbero-venv +source ./cerbero-venv/bin/activate +``` + +## Clone and navigate to repository +``` +git clone https://github.com/stikdev/cerbero +cd cerbero +``` + +## Bootstrap cerbero +``` +# WARNING: cerbero-python3.sh temporarily replaces /usr/bin/python and +# /usr/bin/python3 with symlinks to the virtual env python3 path. +# This is because cerbero invokes python3 four different ways: +# 'python3' +# 'python' +# '/usr/bin/python3' +# '/usr/bin/python' +# All four of these invokations MUST link to the same python3 executable. + +./cerbero-python3.sh bootstrap +``` + +## Build gstreamer +``` +# The package (build) process has two steps: First it compiles all of +# gstreamer's dependencies, which will take up to 8 hours, and second it will +# package the dependencies into rpm files that are added to the home +# directory, which will take up to 20 minutes. +# Dependencies are only recompiled and repackaged on an as-needed basis. + +./cerbero-python3.sh package gstreamer-1.0 +``` + +# Begin original readme: + # Description Cerbero is a cross-platform build aggregator for Open Source projects that builds diff --git a/cerbero-python3.sh b/cerbero-python3.sh new file mode 100644 index 000000000..b6e944fa1 --- /dev/null +++ b/cerbero-python3.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Ensure that the virtual environment is activated +[ -z "$VIRTUAL_ENV" ] && echo "You must activate the 'cerbero-venv' virtual environment (i.e. \`source /path/to/cerbero-venv/bin/activate\`)" && exit 1; + +echo "WARNING: This script will temporarily replace /usr/bin/python and /usr/bin/python3 with symlinks to the virtual environment python3 path." + +sudo mv /usr/bin/python /usr/bin/pythonbak || { echo 'Cannot move /usr/bin/python'; exit 1; } +sudo ln $(which python) /usr/bin/python || { echo "Cannot link $(which python) to /usr/bin/python"; exit 1; } + +if [ -f '/usr/bin/python3' ]; then + sudo mv /usr/bin/python3 /usr/bin/python3bak || { echo 'Cannot move /usr/bin/python3'; exit 1; } +fi + +sudo ln $(which python) /usr/bin/python3 || { echo "Cannot link $(which python) to /usr/bin/python3"; exit 1; } + +./cerbero-uninstalled "$@" + +sudo rm /usr/bin/python + +sudo mv /usr/bin/pythonbak /usr/bin/python + +sudo rm /usr/bin/python3 + +if [ -f '/usr/bin/python3bak' ]; then + sudo mv /usr/bin/python3bak /usr/bin/python3 +fi diff --git a/cerbero/packages/disttarball.py b/cerbero/packages/disttarball.py index 163cdd01e..badd462c8 100644 --- a/cerbero/packages/disttarball.py +++ b/cerbero/packages/disttarball.py @@ -181,7 +181,7 @@ def _write_tar(self, filename, package_prefix, files): else: tar_cmd += ['--bzip2'] elif self.compress == 'xz': - tar_cmd += ['--use-compress-program=xz --threads=0'] + tar_cmd += ['--use-compress-program=xz'] else: raise AssertionError("Unknown tar compression: {}".format(self.compress)) try: diff --git a/cerbero/packages/rpm.py b/cerbero/packages/rpm.py index 219558242..94e3aeaa1 100644 --- a/cerbero/packages/rpm.py +++ b/cerbero/packages/rpm.py @@ -18,6 +18,7 @@ import os import shutil +import sys import tempfile from cerbero.config import Architecture @@ -289,13 +290,25 @@ def _get_requires(self, package_type): def _files_list(self, package_type): if isinstance(self.package, MetaPackage): return '' + + version_tag = sys.version_info.major.toString() + sys.version_info.minor.toString() + files = self.files_list(package_type) for f in [x for x in files if x.endswith('.py')]: - if f + 'c' not in files: - files.append(f + 'c') - if f + 'o' not in files: - files.append(f + 'o') - return '\n'.join([os.path.join('%{prefix}', x) for x in files]) + filepath, filename = os.path.split(f) + pycache_path = os.path.join(filepath, '__pycache__') + + pyc_filename = os.path.splitext(filename)[0] + '.cpython' + version_tag + '.pyc' + pyc_path = os.path.join(pycache_path, pyc_filename) + if pyc_path not in files: + files.append(pyc_path) + + pyo_filename = os.path.splitext(filename)[0] + '.cpython' + version_tag + '.opt-1.pyc' + pyo_path = os.path.join(pycache_path, pyo_filename) + if pyo_path not in files: + files.append(pyo_path) + + return '\n'.join([os.path.join('%{prefix}', x) for x in files]) def _devel_package_and_files(self): args = {}