Fix: Amazon Linux builds

README.md

@@ -1,3 +1,68 @@
+# Waymark Readme
+
+Cerbero is the build system for gstreamer. Cerbero requires some fixes to run properly on Amazon Linux, which are included in this repository.
+
+Here are the modified build instructions for Amazon Linux. The original Cerbero readme is further below.
+
+## Requirements
+
+ - Amazon Linux 2 EC2 instance or [virtual machine](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/amazon-linux-2-virtual-machine.html)
+ - 12gb available disk space. WARNING: The default EBS volume size for many EC2 instance types (8gb) is too small.
+ - 4gb ram
+
+## Install dependencies
+```
+# Ensure system is up-to-date
+sudo yum update
+sudo reboot
+
+# Install python 3, git, and alsa-lib-devel.
+# alsa-lib-devel is a gstreamer dependency that cerbero will fail to install,
+# likely because the alsa dev package has many different names on different
+# platforms. cerbero generally succeeds at automatically installing other
+# dependencies.
+sudo yum install python3 git alsa-lib-devel
+```
+
+## Create and activate python 3 virtual environment
+```
+python3 -m venv cerbero-venv
+source ./cerbero-venv/bin/activate
+```
+
+## Clone and navigate to repository
+```
+git clone https://github.com/stikdev/cerbero
+cd cerbero
+```
+
+## Bootstrap cerbero
+```
+# WARNING: cerbero-python3.sh temporarily replaces /usr/bin/python and
+# /usr/bin/python3 with symlinks to the virtual env python3 path.
+# This is because cerbero invokes python3 four different ways:
+#   'python3'
+#   'python'
+#   '/usr/bin/python3'
+#   '/usr/bin/python'
+# All four of these invokations MUST link to the same python3 executable.
+
+./cerbero-python3.sh bootstrap
+```
+
+## Build gstreamer
+```
+# The package (build) process has two steps: First it compiles all of
+# gstreamer's dependencies, which will take up to 8 hours, and second it will
+# package the dependencies into rpm files that are added to the home
+# directory, which will take up to 20 minutes.
+# Dependencies are only recompiled and repackaged on an as-needed basis.
+
+./cerbero-python3.sh package gstreamer-1.0
+```
+
+# Begin original readme:
+
 # Description
 
 Cerbero is a cross-platform build aggregator for Open Source projects that builds

cerbero-python3.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+# Ensure that the virtual environment is activated
+[ -z "$VIRTUAL_ENV" ] && echo "You must activate the 'cerbero-venv' virtual environment (i.e. \`source /path/to/cerbero-venv/bin/activate\`)" && exit 1;
+
+echo "WARNING: This script will temporarily replace /usr/bin/python and /usr/bin/python3 with symlinks to the virtual environment python3 path."
+
+sudo mv /usr/bin/python /usr/bin/pythonbak || { echo 'Cannot move /usr/bin/python'; exit 1; }
+sudo ln $(which python) /usr/bin/python || { echo "Cannot link $(which python) to /usr/bin/python"; exit 1; }
+
+if [ -f '/usr/bin/python3' ]; then
+  sudo mv /usr/bin/python3 /usr/bin/python3bak || { echo 'Cannot move /usr/bin/python3'; exit 1; }
+fi
+
+sudo ln $(which python) /usr/bin/python3 || { echo "Cannot link $(which python) to /usr/bin/python3"; exit 1; }
+
+./cerbero-uninstalled "$@"
+
+sudo rm /usr/bin/python
+
+sudo mv /usr/bin/pythonbak /usr/bin/python
+
+sudo rm /usr/bin/python3
+
+if [ -f '/usr/bin/python3bak' ]; then
+  sudo mv /usr/bin/python3bak /usr/bin/python3
+fi

cerbero/packages/disttarball.py

@@ -181,7 +181,7 @@ def _write_tar(self, filename, package_prefix, files):
             else:
                 tar_cmd += ['--bzip2']
         elif self.compress == 'xz':
-            tar_cmd += ['--use-compress-program=xz --threads=0']
+            tar_cmd += ['--use-compress-program=xz']
         else:
             raise AssertionError("Unknown tar compression: {}".format(self.compress))
         try:

cerbero/packages/rpm.py

@@ -18,6 +18,7 @@
 
 import os
 import shutil
+import sys
 import tempfile
 
 from cerbero.config import Architecture
@@ -289,13 +290,25 @@ def _get_requires(self, package_type):
     def _files_list(self, package_type):
         if isinstance(self.package, MetaPackage):
             return ''
+
+        version_tag = sys.version_info.major.toString() + sys.version_info.minor.toString()
+
         files = self.files_list(package_type)
         for f in [x for x in files if x.endswith('.py')]:
-            if f + 'c' not in files:
-                files.append(f + 'c')
-            if f + 'o' not in files:
-                files.append(f + 'o')
-        return '\n'.join([os.path.join('%{prefix}',  x) for x in files])
+            filepath, filename = os.path.split(f)
+            pycache_path = os.path.join(filepath, '__pycache__')
+
+            pyc_filename = os.path.splitext(filename)[0] + '.cpython' + version_tag + '.pyc'
+            pyc_path = os.path.join(pycache_path, pyc_filename)
+            if pyc_path not in files:
+                files.append(pyc_path)
+
+            pyo_filename = os.path.splitext(filename)[0] + '.cpython' + version_tag + '.opt-1.pyc'
+            pyo_path = os.path.join(pycache_path, pyo_filename)
+            if pyo_path not in files:
+                files.append(pyo_path)
+
+        return '\n'.join([os.path.join('%{prefix}', x) for x in files])
 
     def _devel_package_and_files(self):
         args = {}