shotgunsoftware · j0yu · Jun 6, 2019 · Jun 6, 2019 · Jun 6, 2019 · Jun 7, 2019
diff --git a/Dockerfile b/Dockerfile
@@ -6,6 +6,8 @@
 #
 
 # dockerfile to spin up a container suitable for generation of documentation
+# - Gem/pip files used for setup are located in /usr/local/src
+# - The default build_docs.sh is in /usr/local/src/scripts
 
 FROM centos:7
 LABEL maintainer="toolkit@shotgunsoftware.com"
@@ -18,58 +20,69 @@ LABEL maintainer="toolkit@shotgunsoftware.com"
 RUN yum update -y && \
     yum install -y epel-release && \
     yum install -y \
-    # Generic build packages
-    autoconf \
-    automake \
-    gcc \
-    gcc-c++ \
-    git \
-    libtool \
-    make \
-    nasm \
-    perl-devel \
-    zlib-devel \
-    tar \
-    nc \
-    xz \
-    # sphinx
-    pandoc \
-    # Python libs
-    python-pip \
-    # Ruby
-    libyaml-devel \
-    openssl-devel \
-    libreadline-dev \
-    zlib-devel \
-    python-pyside \
+        # Generic build packages
+        # autoconf, automake, gcc will be installed as libtool dependencies
+        gcc-c++ \
+        git \
+        libtool \
+        make \
+        nasm \
+        nc \
+        perl-devel \
+        # sphinx
+        pandoc \
+        # Python libs
+        python-pip \
+        python-pyside \
+        # Ruby
+        libreadline-dev \
+        libyaml-devel \
+        openssl-devel \
+        zlib-devel && \
     yum clean all
 
 # Ruby
 ENV RUBY_MAJOR_VER=2.5 \
-    RUBY_VER=2.5.3 \
-    PATH=/opt/ruby-2.5/bin:$PATH
+    RUBY_VER=2.5.3
+ENV PATH=/opt/ruby-${RUBY_MAJOR_VER}/bin:$PATH
 RUN DIR=$(mktemp -d) && cd ${DIR} && \
-    curl -OLs http://ftp.ruby-lang.org/pub/ruby/ruby-$RUBY_VER.tar.gz  && \
-    tar xzf ruby-$RUBY_VER.tar.gz && \
-    cd ruby-$RUBY_VER && \
-    ./configure --prefix=/opt/ruby-${RUBY_VER} --bindir=/opt/ruby-${RUBY_VER}/bin --disable-install-doc && \
-    make -j 8 && \
+    curl -Ls http://ftp.ruby-lang.org/pub/ruby/ruby-${RUBY_VER}.tar.gz | \
+        tar --strip-components=1 -xz && \
+    ./configure \
+        --prefix=/opt/ruby-${RUBY_VER} \
+        --bindir=/opt/ruby-${RUBY_VER}/bin \
+        --disable-install-doc && \
+    make -j $(nproc) && \
     make install && \
     make distclean && \
     rm -rf ${DIR} && \
     cd /opt/ && ln -sfn ruby-${RUBY_VER} ruby-${RUBY_MAJOR_VER}
 
-RUN mkdir -p /app
-WORKDIR /app
-
-COPY ./Gemfile /app
-COPY ./Gemfile.lock /app
-COPY ./requirements.txt /app
+# Copy in any Gem/pip related files
+WORKDIR /usr/local/src
+COPY ./Gemfile .
+COPY ./Gemfile.lock .
+COPY ./requirements.txt .
 
 # Install any needed packages specified in requirements.txt
 RUN pip install -r requirements.txt
 
 # note: stay on bundler 1.17 to be travis compatible
-RUN gem install bundler -v 1.17.2 --no-document
-RUN bundle config --global jobs 7
-RUN bundle install
+RUN gem install bundler -v 1.17.2 --no-document && \
+    bundle config --global jobs $(( $(nproc) - 1 )) && \
+    bundle install
+
+# Copy in and set "build_docs.sh" as default image executable for "docker run"
+COPY ./jekyll jekyll
+COPY ./scripts scripts
+COPY ./sphinx sphinx
+ENV PATH=${PATH}:/usr/local/src/scripts
+
+# Install tk-core (python only)
+RUN get-tk-core-packages.sh /usr/local/lib64/python
+ENV PYTHONPATH=${PYTHONPATH}:/usr/local/lib64/python
+
+RUN mkdir -vp _doc_generator_tmp/markdown_src
+VOLUME [ "$(pwd)/_doc_generator_tmp/markdown_src" ]
+
+ENTRYPOINT [ "serve_docs.sh" ]
diff --git a/preview_docs.sh b/preview_docs.sh
@@ -7,12 +7,172 @@
 # this software in either electronic or hard copy form.
 #
 
-# Builds a local preview of the documentation in the parent repository that is
-# including tk-doc-generator as a submodule.
+set -eu -o pipefail
 
+
+ARG1="${1:-}"
+case "${ARG1}" in
+    -h|--help)
+        cat << EOF
+Usage:  preview_docs.sh [--rebuild|-h] [URL_PATH]
+
+Builds a local preview of the documentation in the parent repository that
+includes tk-doc-generator as a sub-module.
+
+Options:
+    --rebuild       Force re-build docker image. By default (for speed),
+                    it will only build a new Docker image if one doesn't exist.
+    -h, --help      Print this help and exit 0.
+    URL_PATH        Explicit URL sub folder/path to host under.
+                    By default, it's automatically calculated in order:
+                    1. DOC_PATH=/* in .travis.yml
+                    2. Current folder name
+
+Environment variables:
+    IMAGE_TAG:
+        Tag for the image Docker builds.
+        Defaults "tk-doc-generator" if not set
+
+    MOUNT_TO:
+        Destination directory which the parent directory this script,
+        the folder above tk-doc-generator, is mounted to in the container.
+        Defaults "/app" if not set
+
+Examples:
+    Typically, this command is run in the parent repository that includes
+    tk-doc-generator as a sub-module.
+
+    To force an image rebuild by running before creating documentation:
+
+        tk-doc-generator/preview_docs.sh --rebuild
+
+    Some environment variables will also influence this script's behaviour.
+    For example, you can run the script like this to build/run the docker image
+    tagged "my-doc-generator", you can run:
+
+        IMAGE_TAG="my-doc-generator" tk-doc-generator/preview_docs.sh
+
+EOF
+        exit
+    ;;
+esac
+
+# Setup useful directory variables for current script's folder/parent folder
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-docker build ${THIS_DIR}/../tk-doc-generator -t tk-doc-generator && \
-docker run --mount type=bind,source="${THIS_DIR}/..",target=/app tk-doc-generator \
-/app/tk-doc-generator/scripts/build_docs.sh --url=http://localhost --url-path=${THIS_DIR}/../_build --source=/app/docs --output=/app/_build && \
-echo "Documentation built in ${THIS_DIR}/../_build/index.html"
+
+# Build if image does not exist OR first argument is --rebuild
+IMAGE_TAG="${IMAGE_TAG:-tk-doc-generator}"
+if [ -z "$(docker images --quiet ${IMAGE_TAG})" -o "${ARG1}" == "--rebuild" ]
+then
+    docker build --tag "${IMAGE_TAG}" ${THIS_DIR}
+    shift  # Process remaining arg as explicit URL_PATH
+else
+    echo "'${IMAGE_TAG}' docker image already built"
+    echo "To force a re-build, try running this script with --rebuild flag"
+    echo
+fi
+
+
+# Setup mount/volume flags depending on which one current docker supports
+MOUNT_TO="${MOUNT_TO:-/app}"
+if ( docker run --help | grep --quiet -- --mount )
+then
+    MOUNT_FLAGS='--mount '"type=bind,source=$(pwd),target=${MOUNT_TO}"
+else
+    MOUNT_FLAGS='-v '"$(pwd):${MOUNT_TO}"
+fi
+
+
+# Finally, perform the document generation in container
+EXPOSED_PORT="4000"
+URL_PATH="${1:-$(
+    grep -oP '(?<=DOC_PATH=/)[a-zA-Z0-9][a-zA-Z0-9_.-]*' $(pwd)/.travis.yml \
+    || basename $(pwd)
+)}"
+
+if (echo "${URL_PATH}" | grep -qP '^[a-zA-Z0-9][a-zA-Z0-9_.-]*$')
+then
+    while true
+    do
+        CONTAINER_NAME="${URL_PATH}-${EXPOSED_PORT}"
+        CONTAINER_ID=$(docker run --rm -d \
+            --hostname="${CONTAINER_NAME}" \
+            --name="${CONTAINER_NAME}" \
+            -p "${EXPOSED_PORT}:${EXPOSED_PORT}" \
+            -e EXPOSED_PORT="${EXPOSED_PORT}" \
+            -e URLPATH="/${URL_PATH}" \
+            -e SOURCE="${MOUNT_TO}/docs" \
+            -e OUTPUT="${MOUNT_TO}/_build" \
+            ${MOUNT_FLAGS} \
+            ${IMAGE_TAG}) \
+        && break \
+        || EXPOSED_PORT="$(( EXPOSED_PORT + 1 ))"
+    done
+else
+    echo "Invalid URL_PATH to host as: '${URL_PATH}'"
+    echo 'It must only contain these characters: [a-zA-Z0-9_.-]'
+    echo
+    echo 'Please either:'
+    echo '1. Set a valid DOC_PATH=/something in $(pwd)/.travis.yml, where that'
+    echo '   something must only contain those valid characters:  e.g.'
+    echo
+    echo "       env:"
+    echo "         global:"
+    echo "             - DOC_PATH=/tk-amazing-tool"
+    echo
+    echo "2. Rename current folder to only contain those valid characters."
+    echo
+    echo "3. Explicitly provide a valid URL_PATH to create under, e.g."
+    echo
+    echo "       preview_docs.sh tk-amazing-tool"
+    echo
+    echo
+    URL_PATH=''
+    while true
+    do
+        CONTAINER_ID=$(docker run --rm -d \
+                -p "${EXPOSED_PORT}:${EXPOSED_PORT}" \
+                -e EXPOSED_PORT="${EXPOSED_PORT}" \
+                -e SOURCE="${MOUNT_TO}/docs" \
+                -e OUTPUT="${MOUNT_TO}/_build" \
+                ${MOUNT_FLAGS} \
+                ${IMAGE_TAG}) \
+            && break \
+            || EXPOSED_PORT="$(( EXPOSED_PORT + 1 ))"
+    done
+    CONTAINER_NAME=$(docker ps -f id=${CONTAINER_ID} --format '{{.Names}}')
+fi
+echo "Waiting for container to startup: ${CONTAINER_NAME}"
+echo
+echo
+
+# To prevent error in "docker logs" writing out blank logs file
+# we use a buffer file so: docker logs -> buffer -> log file
+TMP_LOG=$(mktemp)
+TMP_LOG_BUFFER=$(mktemp)
+while
+    docker inspect ${CONTAINER_ID} &> /dev/null \
+    && docker logs ${CONTAINER_ID} > ${TMP_LOG_BUFFER}
+do
+    cp -f ${TMP_LOG_BUFFER} ${TMP_LOG}
+    grep -q "Server running... press ctrl-c to stop." ${TMP_LOG} && break
+done
+
+# Output logs so users can see if it failed during startup
+cat ${TMP_LOG}
+
+# Output "Open in web-browser" if the serve script actually succeeded
+docker inspect ${CONTAINER_ID} &> /dev/null && cat << EOF
+###############################################################################
+
+      Documentation built. Open in web-browser:
+
+  http://localhost:${EXPOSED_PORT}/${URL_PATH}${URL_PATH:+/}
+
+         VIEW logs: docker logs -f ${CONTAINER_NAME}
+CONVERT sphinx rst: docker exec ${CONTAINER_NAME} build_sphinx.py ${MOUNT_TO}/docs
+  CHECK if running: docker ps --filter name=${CONTAINER_NAME}
+    STOP container: docker stop ${CONTAINER_NAME}
+
+EOF
diff --git a/scripts/build_sphinx.py b/scripts/build_sphinx.py
@@ -1,3 +1,4 @@
+#!/bin/python2
 # Copyright 2019 Autodesk, Inc.  All rights reserved.
 #
 # Use of this software is subject to the terms of the Autodesk license agreement

diff --git a/scripts/get-tk-core-packages.sh b/scripts/get-tk-core-packages.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# Copyright 2019 Autodesk, Inc.  All rights reserved.
+#
+# Use of this software is subject to the terms of the Autodesk license agreement
+# provided at the time of installation or download, or which otherwise accompanies
+# this software in either electronic or hard copy form.
+#
+
+set -eu -o pipefail
+
+usage() {
+cat << EOF
+Usage:  get-tk-core-packages.sh [-h] FOLDER
+
+Downloads the latest (tagged) version of tk-core into a given FOLDER.
+If FOLDER does not exist, it will try create it.
+
+Options:
+    -h, --help      Print this help and exit 0
+
+Example:
+    To download the latest sgtk, tank and tank_vendor package folders into
+    /usr/local/lib64/python, run:
+
+        get-tk-core-packages.sh /usr/local/lib64/python
+
+    This should result in a sgtk, tank and tank_vendor directly under
+    /usr/local/lib64/python.
+EOF
+}
+
+if [ $# -ne 1 ]
+then
+    echo 'ERROR: Please provide a folder to download/install into!'
+    echo
+    usage
+    exit 1
+fi
+
+case $1 in
+    -h|--help)
+        usage
+        exit
+    ;;
+esac
+
+# Try create folder if it does not currently exist
+FOLDER="$1"
+[ -d "${FOLDER}" ] || mkdir -vp "${FOLDER}"
+
+curl_untar() {
+    # GitHub API to get all tags (seems to be latest first/at top of file)
+    TAGS_FILE="$(mktemp)"
+    curl -s "https://api.github.com/repos/shotgunsoftware/tk-core/tags" \
+    | tee ${TAGS_FILE}
+
+    # Grab first version (tag name), extract sgtk, tank and tank_vendor packages
+    VERSION=$(grep --color=never -m 1 -o -P '(?<="name": ")[^"]+' ${TAGS_FILE})
+
+    echo "Downloading and extracting tk-core ${VERSION} into: ${FOLDER}"
+    curl -Ls https://github.com/shotgunsoftware/tk-core/archive/${VERSION}.tar.gz \
+    | tar --directory "${FOLDER}" \
+        --extract --gzip --file=- \
+        --exclude='*tests*' --wildcards --wildcards-match-slash \
+        --strip-components=2 '*python*'
+}
+
+temp_repo() {
+    # Clone into a temporary directory
+    TEMP_REPO="$(mktemp -d)"
+    pushd "${TEMP_REPO}"
+    git clone --no-checkout https://github.com/shotgunsoftware/tk-core.git .
+
+    # Checkout latest version, grab the files/folders under python folder
+    VERSION="$(git for-each-ref --sort=taggerdate --format '%(tag)' | tail -1)"
+    echo "Cloned and extracting tk-core ${VERSION} into: ${FOLDER}"
+    git checkout "${VERSION}"
+    mv python/* "${FOLDER}"
+    popd
+}
+
+# Try faster curl_untar first before falling back to temporary clone method
+curl_untar || temp_repo
+ls -lh "${FOLDER}"