diff --git a/Dockerfile b/Dockerfile index 1052adf..026b11e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ # # dockerfile to spin up a container suitable for generation of documentation +# - Gem/pip files used for setup are located in /usr/local/src +# - The default build_docs.sh is in /usr/local/src/scripts FROM centos:7 LABEL maintainer="toolkit@shotgunsoftware.com" @@ -18,58 +20,69 @@ LABEL maintainer="toolkit@shotgunsoftware.com" RUN yum update -y && \ yum install -y epel-release && \ yum install -y \ - # Generic build packages - autoconf \ - automake \ - gcc \ - gcc-c++ \ - git \ - libtool \ - make \ - nasm \ - perl-devel \ - zlib-devel \ - tar \ - nc \ - xz \ - # sphinx - pandoc \ - # Python libs - python-pip \ - # Ruby - libyaml-devel \ - openssl-devel \ - libreadline-dev \ - zlib-devel \ - python-pyside \ + # Generic build packages + # autoconf, automake, gcc will be installed as libtool dependencies + gcc-c++ \ + git \ + libtool \ + make \ + nasm \ + nc \ + perl-devel \ + # sphinx + pandoc \ + # Python libs + python-pip \ + python-pyside \ + # Ruby + libreadline-dev \ + libyaml-devel \ + openssl-devel \ + zlib-devel && \ yum clean all # Ruby ENV RUBY_MAJOR_VER=2.5 \ - RUBY_VER=2.5.3 \ - PATH=/opt/ruby-2.5/bin:$PATH + RUBY_VER=2.5.3 +ENV PATH=/opt/ruby-${RUBY_MAJOR_VER}/bin:$PATH RUN DIR=$(mktemp -d) && cd ${DIR} && \ - curl -OLs http://ftp.ruby-lang.org/pub/ruby/ruby-$RUBY_VER.tar.gz && \ - tar xzf ruby-$RUBY_VER.tar.gz && \ - cd ruby-$RUBY_VER && \ - ./configure --prefix=/opt/ruby-${RUBY_VER} --bindir=/opt/ruby-${RUBY_VER}/bin --disable-install-doc && \ - make -j 8 && \ + curl -Ls http://ftp.ruby-lang.org/pub/ruby/ruby-${RUBY_VER}.tar.gz | \ + tar --strip-components=1 -xz && \ + ./configure \ + --prefix=/opt/ruby-${RUBY_VER} \ + --bindir=/opt/ruby-${RUBY_VER}/bin \ + --disable-install-doc && \ + make -j $(nproc) && \ make install && \ make distclean && \ rm -rf ${DIR} && \ cd /opt/ && ln -sfn ruby-${RUBY_VER} ruby-${RUBY_MAJOR_VER} -RUN mkdir -p /app -WORKDIR /app - -COPY ./Gemfile /app -COPY ./Gemfile.lock /app -COPY ./requirements.txt /app +# Copy in any Gem/pip related files +WORKDIR /usr/local/src +COPY ./Gemfile . +COPY ./Gemfile.lock . +COPY ./requirements.txt . # Install any needed packages specified in requirements.txt RUN pip install -r requirements.txt # note: stay on bundler 1.17 to be travis compatible -RUN gem install bundler -v 1.17.2 --no-document -RUN bundle config --global jobs 7 -RUN bundle install +RUN gem install bundler -v 1.17.2 --no-document && \ + bundle config --global jobs $(( $(nproc) - 1 )) && \ + bundle install + +# Copy in and set "build_docs.sh" as default image executable for "docker run" +COPY ./jekyll jekyll +COPY ./scripts scripts +COPY ./sphinx sphinx +ENV PATH=${PATH}:/usr/local/src/scripts + +# Install tk-core (python only) +RUN get-tk-core-packages.sh /usr/local/lib64/python +ENV PYTHONPATH=${PYTHONPATH}:/usr/local/lib64/python + +RUN mkdir -vp _doc_generator_tmp/markdown_src +VOLUME [ "$(pwd)/_doc_generator_tmp/markdown_src" ] + +ENTRYPOINT [ "serve_docs.sh" ] \ No newline at end of file diff --git a/preview_docs.sh b/preview_docs.sh index 453ec2b..c918a33 100755 --- a/preview_docs.sh +++ b/preview_docs.sh @@ -7,12 +7,172 @@ # this software in either electronic or hard copy form. # -# Builds a local preview of the documentation in the parent repository that is -# including tk-doc-generator as a submodule. +set -eu -o pipefail + +ARG1="${1:-}" +case "${ARG1}" in + -h|--help) + cat << EOF +Usage: preview_docs.sh [--rebuild|-h] [URL_PATH] + +Builds a local preview of the documentation in the parent repository that +includes tk-doc-generator as a sub-module. + +Options: + --rebuild Force re-build docker image. By default (for speed), + it will only build a new Docker image if one doesn't exist. + -h, --help Print this help and exit 0. + URL_PATH Explicit URL sub folder/path to host under. + By default, it's automatically calculated in order: + 1. DOC_PATH=/* in .travis.yml + 2. Current folder name + +Environment variables: + IMAGE_TAG: + Tag for the image Docker builds. + Defaults "tk-doc-generator" if not set + + MOUNT_TO: + Destination directory which the parent directory this script, + the folder above tk-doc-generator, is mounted to in the container. + Defaults "/app" if not set + +Examples: + Typically, this command is run in the parent repository that includes + tk-doc-generator as a sub-module. + + To force an image rebuild by running before creating documentation: + + tk-doc-generator/preview_docs.sh --rebuild + + Some environment variables will also influence this script's behaviour. + For example, you can run the script like this to build/run the docker image + tagged "my-doc-generator", you can run: + + IMAGE_TAG="my-doc-generator" tk-doc-generator/preview_docs.sh + +EOF + exit + ;; +esac + +# Setup useful directory variables for current script's folder/parent folder THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -docker build ${THIS_DIR}/../tk-doc-generator -t tk-doc-generator && \ -docker run --mount type=bind,source="${THIS_DIR}/..",target=/app tk-doc-generator \ -/app/tk-doc-generator/scripts/build_docs.sh --url=http://localhost --url-path=${THIS_DIR}/../_build --source=/app/docs --output=/app/_build && \ -echo "Documentation built in ${THIS_DIR}/../_build/index.html" + +# Build if image does not exist OR first argument is --rebuild +IMAGE_TAG="${IMAGE_TAG:-tk-doc-generator}" +if [ -z "$(docker images --quiet ${IMAGE_TAG})" -o "${ARG1}" == "--rebuild" ] +then + docker build --tag "${IMAGE_TAG}" ${THIS_DIR} + shift # Process remaining arg as explicit URL_PATH +else + echo "'${IMAGE_TAG}' docker image already built" + echo "To force a re-build, try running this script with --rebuild flag" + echo +fi + + +# Setup mount/volume flags depending on which one current docker supports +MOUNT_TO="${MOUNT_TO:-/app}" +if ( docker run --help | grep --quiet -- --mount ) +then + MOUNT_FLAGS='--mount '"type=bind,source=$(pwd),target=${MOUNT_TO}" +else + MOUNT_FLAGS='-v '"$(pwd):${MOUNT_TO}" +fi + + +# Finally, perform the document generation in container +EXPOSED_PORT="4000" +URL_PATH="${1:-$( + grep -oP '(?<=DOC_PATH=/)[a-zA-Z0-9][a-zA-Z0-9_.-]*' $(pwd)/.travis.yml \ + || basename $(pwd) +)}" + +if (echo "${URL_PATH}" | grep -qP '^[a-zA-Z0-9][a-zA-Z0-9_.-]*$') +then + while true + do + CONTAINER_NAME="${URL_PATH}-${EXPOSED_PORT}" + CONTAINER_ID=$(docker run --rm -d \ + --hostname="${CONTAINER_NAME}" \ + --name="${CONTAINER_NAME}" \ + -p "${EXPOSED_PORT}:${EXPOSED_PORT}" \ + -e EXPOSED_PORT="${EXPOSED_PORT}" \ + -e URLPATH="/${URL_PATH}" \ + -e SOURCE="${MOUNT_TO}/docs" \ + -e OUTPUT="${MOUNT_TO}/_build" \ + ${MOUNT_FLAGS} \ + ${IMAGE_TAG}) \ + && break \ + || EXPOSED_PORT="$(( EXPOSED_PORT + 1 ))" + done +else + echo "Invalid URL_PATH to host as: '${URL_PATH}'" + echo 'It must only contain these characters: [a-zA-Z0-9_.-]' + echo + echo 'Please either:' + echo '1. Set a valid DOC_PATH=/something in $(pwd)/.travis.yml, where that' + echo ' something must only contain those valid characters: e.g.' + echo + echo " env:" + echo " global:" + echo " - DOC_PATH=/tk-amazing-tool" + echo + echo "2. Rename current folder to only contain those valid characters." + echo + echo "3. Explicitly provide a valid URL_PATH to create under, e.g." + echo + echo " preview_docs.sh tk-amazing-tool" + echo + echo + URL_PATH='' + while true + do + CONTAINER_ID=$(docker run --rm -d \ + -p "${EXPOSED_PORT}:${EXPOSED_PORT}" \ + -e EXPOSED_PORT="${EXPOSED_PORT}" \ + -e SOURCE="${MOUNT_TO}/docs" \ + -e OUTPUT="${MOUNT_TO}/_build" \ + ${MOUNT_FLAGS} \ + ${IMAGE_TAG}) \ + && break \ + || EXPOSED_PORT="$(( EXPOSED_PORT + 1 ))" + done + CONTAINER_NAME=$(docker ps -f id=${CONTAINER_ID} --format '{{.Names}}') +fi +echo "Waiting for container to startup: ${CONTAINER_NAME}" +echo +echo + +# To prevent error in "docker logs" writing out blank logs file +# we use a buffer file so: docker logs -> buffer -> log file +TMP_LOG=$(mktemp) +TMP_LOG_BUFFER=$(mktemp) +while + docker inspect ${CONTAINER_ID} &> /dev/null \ + && docker logs ${CONTAINER_ID} > ${TMP_LOG_BUFFER} +do + cp -f ${TMP_LOG_BUFFER} ${TMP_LOG} + grep -q "Server running... press ctrl-c to stop." ${TMP_LOG} && break +done + +# Output logs so users can see if it failed during startup +cat ${TMP_LOG} + +# Output "Open in web-browser" if the serve script actually succeeded +docker inspect ${CONTAINER_ID} &> /dev/null && cat << EOF +############################################################################### + + Documentation built. Open in web-browser: + + http://localhost:${EXPOSED_PORT}/${URL_PATH}${URL_PATH:+/} + + VIEW logs: docker logs -f ${CONTAINER_NAME} +CONVERT sphinx rst: docker exec ${CONTAINER_NAME} build_sphinx.py ${MOUNT_TO}/docs + CHECK if running: docker ps --filter name=${CONTAINER_NAME} + STOP container: docker stop ${CONTAINER_NAME} + +EOF \ No newline at end of file diff --git a/scripts/build_sphinx.py b/scripts/build_sphinx.py index 34c0dea..3127cad 100755 --- a/scripts/build_sphinx.py +++ b/scripts/build_sphinx.py @@ -1,3 +1,4 @@ +#!/bin/python2 # Copyright 2019 Autodesk, Inc. All rights reserved. # # Use of this software is subject to the terms of the Autodesk license agreement diff --git a/scripts/get-tk-core-packages.sh b/scripts/get-tk-core-packages.sh new file mode 100755 index 0000000..b966142 --- /dev/null +++ b/scripts/get-tk-core-packages.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Copyright 2019 Autodesk, Inc. All rights reserved. +# +# Use of this software is subject to the terms of the Autodesk license agreement +# provided at the time of installation or download, or which otherwise accompanies +# this software in either electronic or hard copy form. +# + +set -eu -o pipefail + +usage() { +cat << EOF +Usage: get-tk-core-packages.sh [-h] FOLDER + +Downloads the latest (tagged) version of tk-core into a given FOLDER. +If FOLDER does not exist, it will try create it. + +Options: + -h, --help Print this help and exit 0 + +Example: + To download the latest sgtk, tank and tank_vendor package folders into + /usr/local/lib64/python, run: + + get-tk-core-packages.sh /usr/local/lib64/python + + This should result in a sgtk, tank and tank_vendor directly under + /usr/local/lib64/python. +EOF +} + +if [ $# -ne 1 ] +then + echo 'ERROR: Please provide a folder to download/install into!' + echo + usage + exit 1 +fi + +case $1 in + -h|--help) + usage + exit + ;; +esac + +# Try create folder if it does not currently exist +FOLDER="$1" +[ -d "${FOLDER}" ] || mkdir -vp "${FOLDER}" + +curl_untar() { + # GitHub API to get all tags (seems to be latest first/at top of file) + TAGS_FILE="$(mktemp)" + curl -s "https://api.github.com/repos/shotgunsoftware/tk-core/tags" \ + | tee ${TAGS_FILE} + + # Grab first version (tag name), extract sgtk, tank and tank_vendor packages + VERSION=$(grep --color=never -m 1 -o -P '(?<="name": ")[^"]+' ${TAGS_FILE}) + + echo "Downloading and extracting tk-core ${VERSION} into: ${FOLDER}" + curl -Ls https://github.com/shotgunsoftware/tk-core/archive/${VERSION}.tar.gz \ + | tar --directory "${FOLDER}" \ + --extract --gzip --file=- \ + --exclude='*tests*' --wildcards --wildcards-match-slash \ + --strip-components=2 '*python*' +} + +temp_repo() { + # Clone into a temporary directory + TEMP_REPO="$(mktemp -d)" + pushd "${TEMP_REPO}" + git clone --no-checkout https://github.com/shotgunsoftware/tk-core.git . + + # Checkout latest version, grab the files/folders under python folder + VERSION="$(git for-each-ref --sort=taggerdate --format '%(tag)' | tail -1)" + echo "Cloned and extracting tk-core ${VERSION} into: ${FOLDER}" + git checkout "${VERSION}" + mv python/* "${FOLDER}" + popd +} + +# Try faster curl_untar first before falling back to temporary clone method +curl_untar || temp_repo +ls -lh "${FOLDER}" \ No newline at end of file diff --git a/scripts/serve_docs.sh b/scripts/serve_docs.sh new file mode 100755 index 0000000..ad7c834 --- /dev/null +++ b/scripts/serve_docs.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Copyright 2019 Autodesk, Inc. All rights reserved. +# +# Use of this software is subject to the terms of the Autodesk license agreement +# provided at the time of installation or download, or which otherwise accompanies +# this software in either electronic or hard copy form. +# + +# Usage: serve_docs.sh +# +# Build and serve documentation using "jekyll serve". +# +# Although this script does not take in arguments directly, certain environment +# variables do change certain settings and paths. +# +# Environment Variables: +# +# SOURCE: Directory to look for documentation to generate from. +# Defaults to current working directory +# (e.g. from "docker run -w DIR DOCKER_IMAGE") +# +# OUTPUT: Directory to output generated HTML and assets in. +# Defaults to "_site" folder under current working directory. +# +# EXPOSED_PORT: Port on host to expose website from. +# Defaults to 4000. +# +# URLPATH: Base directory (under root/domain) to host site on. +# Defaults to /, which resolves to domain root +# e.g. "localhost:4000/". +# + +# exit on error +set -eu -o pipefail + +# Set defaults if no env vars detected +SOURCE=${SOURCE:-$(pwd)} +OUTPUT=${OUTPUT:-$(pwd)/_site} +EXPOSED_PORT=${EXPOSED_PORT:-4000} +URLPATH=${URLPATH:-/} + +# contains Gemfile, jekyll, etc +TK_DOC_GEN_SRC="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )" +OUR_REPO_ROOT="$(readlink -m ${SOURCE}/..)" # Typically user's project root folder + +echo "Cleaning out final build location '${OUTPUT}'..." +rm -rf ${OUTPUT} + +echo "Running Sphinx RST -> Markdown build process..." +build_sphinx.py ${SOURCE} + +# Setup additional plugin paths +PLUGINS="${TK_DOC_GEN_SRC}/jekyll/_plugins" +EXTRA_PLUGIN=${SOURCE}/_plugins +if [ -d "${EXTRA_PLUGIN}" ] +then + echo "Using additional plugins from ${EXTRA_PLUGIN}..." + PLUGINS="${PLUGINS},${EXTRA_PLUGIN}" +fi + +# Setup additional config overrides +CONFIGS="${TK_DOC_GEN_SRC}/jekyll/_config.yml" +OVERRIDE_CONFIG=${OUR_REPO_ROOT}/jekyll_config.yml +if [ -e "${OVERRIDE_CONFIG}" ] +then + echo "Using override config from ${OVERRIDE_CONFIG}..." + CONFIGS="${CONFIGS},${OVERRIDE_CONFIG}" +fi + +echo "Running Jekyll to generate html from markdown..." +echo "---------------------------------------------------" +echo " Source: ${SOURCE}" +echo " Output: ${OUTPUT}" +echo " Plugins: ${PLUGINS}" +echo " Configurations: ${CONFIGS}" +echo " URL (localhost): localhost:${EXPOSED_PORT}${URLPATH}" +echo "---------------------------------------------------" +umask 0000 +echo "$(hostname -I)localhost" >> /etc/hosts + +# Auto-regeneration will be disabled if running server detached. + # --verbose \ +BUNDLE_GEMFILE=${TK_DOC_GEN_SRC}/Gemfile JEKYLL_ENV=production \ +bundle exec jekyll serve \ + --baseurl "${URLPATH}" \ + --host localhost --port "${EXPOSED_PORT}" \ + --trace \ + --config "${CONFIGS}" \ + --plugins "${PLUGINS}" \ + --source "${SOURCE}" \ + --destination "${OUTPUT}" diff --git a/travis-generate-docs.py b/travis-generate-docs.py index f7869e3..ce40006 100755 --- a/travis-generate-docs.py +++ b/travis-generate-docs.py @@ -124,27 +124,30 @@ def main(): current_branch = os.environ.get("TRAVIS_BRANCH") inside_pr = os.environ.get("TRAVIS_PULL_REQUEST") != "false" - # first figure out i we are on master or in a PR. + # first figure out if we are on master or in a PR. if current_branch != "master" or inside_pr: - # we are in a PR. - log.info("Inside a pull request.") + log.info("Inside a pull request (or not master branch).") - # see if we have access to an AWS bucket - if "S3_BUCKET" in os.environ and "S3_WEB_URL" in os.environ: + # See if we have access to an AWS bucket + s3_bucket = os.getenv("S3_BUCKET", default=None) + target_url = os.getenv("S3_WEB_URL", default=None) + if s3_bucket and target_url: log.info("Detected AWS S3 bucket for preview workflow.") - - s3_bucket = os.environ["S3_BUCKET"] - target_url = os.environ["S3_WEB_URL"] - target_url_path = "/tk-doc-generator/{commit}".format( - commit=os.environ["TRAVIS_COMMIT"] - ) - + target_url_path = "/tk-doc-generator/{env[TRAVIS_COMMIT]}" + target_url_path = target_url_path.format(env=os.environ) else: log.warning("No S3_BUCKET and S3_WEB_URL detected in environment. " "No S3 preview will be generated") - s3_bucket = None - # enter dummy paths so we can at least build - # the docs to check for errors + + # Then try use DOC_* for target + target_url = os.getenv("DOC_URL", default=None) + target_url_path = os.getenv("DOC_PATH", default=None) + if target_url and target_url_path: + log.info("Using DOC_URL/PATH.") + else: + # Fall back to using root directly on dummy domain url + log.warning("Using dummy paths so we can at least build the docs " + "to check for errors") target_url = "https://dummy.url.com" target_url_path = "/" @@ -160,12 +163,14 @@ def main(): ) execute_external_command(doc_command) - if s3_bucket: + aws_access_key = os.getenv("AWS_S3_ACCESS_KEY", default=None) + aws_access_token = os.getenv("AWS_S3_ACCESS_TOKEN", default=None) + if s3_bucket and aws_access_key and aws_access_token: log.info("Uploading build result to S3...") s3_client = boto3.client( "s3", - aws_access_key_id=os.environ["AWS_S3_ACCESS_KEY"], - aws_secret_access_key=os.environ["AWS_S3_ACCESS_TOKEN"] + aws_access_key=aws_access_key, + aws_access_token=aws_access_token ) # note: skip the first slash when uploading to S3