diff --git a/Dockerfile b/Dockerfile index 1052adf..9ae880e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ # # dockerfile to spin up a container suitable for generation of documentation +# - Gem/pip files used for setup are located in /usr/local/src +# - The default build_docs.sh is in /usr/local/src/scripts FROM centos:7 LABEL maintainer="toolkit@shotgunsoftware.com" @@ -18,58 +20,66 @@ LABEL maintainer="toolkit@shotgunsoftware.com" RUN yum update -y && \ yum install -y epel-release && \ yum install -y \ - # Generic build packages - autoconf \ - automake \ - gcc \ - gcc-c++ \ - git \ - libtool \ - make \ - nasm \ - perl-devel \ - zlib-devel \ - tar \ - nc \ - xz \ - # sphinx - pandoc \ - # Python libs - python-pip \ - # Ruby - libyaml-devel \ - openssl-devel \ - libreadline-dev \ - zlib-devel \ - python-pyside \ + # Generic build packages + # autoconf, automake, gcc will be installed as libtool dependencies + gcc-c++ \ + git \ + libtool \ + make \ + nasm \ + nc \ + perl-devel \ + # sphinx + pandoc \ + # Python libs + python-pip \ + python-pyside \ + # Ruby + libreadline-dev \ + libyaml-devel \ + openssl-devel \ + zlib-devel && \ yum clean all # Ruby ENV RUBY_MAJOR_VER=2.5 \ - RUBY_VER=2.5.3 \ - PATH=/opt/ruby-2.5/bin:$PATH + RUBY_VER=2.5.3 +ENV PATH=/opt/ruby-${RUBY_MAJOR_VER}/bin:$PATH RUN DIR=$(mktemp -d) && cd ${DIR} && \ - curl -OLs http://ftp.ruby-lang.org/pub/ruby/ruby-$RUBY_VER.tar.gz && \ - tar xzf ruby-$RUBY_VER.tar.gz && \ - cd ruby-$RUBY_VER && \ - ./configure --prefix=/opt/ruby-${RUBY_VER} --bindir=/opt/ruby-${RUBY_VER}/bin --disable-install-doc && \ - make -j 8 && \ + curl -Ls http://ftp.ruby-lang.org/pub/ruby/ruby-${RUBY_VER}.tar.gz | \ + tar --strip-components=1 -xz && \ + ./configure \ + --prefix=/opt/ruby-${RUBY_VER} \ + --bindir=/opt/ruby-${RUBY_VER}/bin \ + --disable-install-doc && \ + make -j $(nproc) && \ make install && \ make distclean && \ rm -rf ${DIR} && \ cd /opt/ && ln -sfn ruby-${RUBY_VER} ruby-${RUBY_MAJOR_VER} -RUN mkdir -p /app -WORKDIR /app - -COPY ./Gemfile /app -COPY ./Gemfile.lock /app -COPY ./requirements.txt /app +# Copy in any Gem/pip related files +WORKDIR /usr/local/src +COPY ./Gemfile . +COPY ./Gemfile.lock . +COPY ./requirements.txt . # Install any needed packages specified in requirements.txt RUN pip install -r requirements.txt # note: stay on bundler 1.17 to be travis compatible -RUN gem install bundler -v 1.17.2 --no-document -RUN bundle config --global jobs 7 -RUN bundle install +RUN gem install bundler -v 1.17.2 --no-document && \ + bundle config --global jobs $(( $(nproc) - 1 )) && \ + bundle install + +# Copy in and set "build_docs.sh" as default image executable for "docker run" +COPY ./jekyll jekyll +COPY ./scripts scripts +COPY ./sphinx sphinx +ENV PATH=${PATH}:/usr/local/src/scripts + +# Install tk-core (python only) +RUN get-tk-core-packages.sh /usr/local/lib64/python +ENV PYTHONPATH=${PYTHONPATH}:/usr/local/lib64/python + +ENTRYPOINT [ "build_docs.sh" ] \ No newline at end of file diff --git a/Gemfile b/Gemfile index dc2aed3..12f59e5 100644 --- a/Gemfile +++ b/Gemfile @@ -10,7 +10,7 @@ source "https://rubygems.org" gem "jekyll", "~> 3.8.5" # This is the default theme for new Jekyll sites. You may change this to anything you like. -gem "just-the-docs", :git => 'https://github.com/shotgunsoftware/just-the-docs.git' +gem "just-the-docs", :git => 'https://github.com/wwfxuk/just-the-docs.git' # If you have any plugins, put them here! group :jekyll_plugins do diff --git a/Gemfile.lock b/Gemfile.lock index 2fffbbc..1969758 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT - remote: https://github.com/shotgunsoftware/just-the-docs.git - revision: 54f8f4d79e2c872402f4b3f75693bb3caba44af3 + remote: https://github.com/wwfxuk/just-the-docs.git + revision: e4d057a8a5f057c6aab101071ef7009e10d4c9e9 specs: just-the-docs (0.2.1) jekyll (~> 3.8.5) diff --git a/docs/_data/en/toc_text.yml b/docs/_data/en/toc_text.yml index f5be72d..b2bdecd 100644 --- a/docs/_data/en/toc_text.yml +++ b/docs/_data/en/toc_text.yml @@ -1,8 +1,9 @@ # Translated text strings used by the table of contents -# +# # For documentation, see https://developer.shotgunsoftware.com/tk-doc-generator/authoring/toc/ # +overview: Overview authoring: Authoring docs markdown-cheatsheet: Markdown Format setting-up: Installation diff --git a/docs/_data/toc.yml b/docs/_data/toc.yml index 59c10d7..afbd309 100644 --- a/docs/_data/toc.yml +++ b/docs/_data/toc.yml @@ -4,6 +4,10 @@ # # +- caption: overview + children: + - page: changelog + - caption: authoring children: - page: authoring diff --git a/docs/en/changelog.md b/docs/en/changelog.md new file mode 100644 index 0000000..1f62e8b --- /dev/null +++ b/docs/en/changelog.md @@ -0,0 +1,55 @@ +--- +layout: default +title: Change Log/History +pagename: changelog +lang: en +--- + +Change Log/History +------------------ + +See also the [GitHub Releases page][releases] + +[releases]: https://github.com/shotgunsoftware/tk-doc-generator/releases + +# v1.x.x (changes from v1.0.0+wwfx.0.2.0 to v1.0.0+wwfx.0.3.1) + +Functional improvements and fixes from testing +`tk-doc-generator` (locally using `preview_docs.sh` and on `travis.ci`) +with `tk-katana` on the [ci-docs](https://github.com/wwfxuk/tk-katana/pull/6) +branch/PR. See also `tk-katana`'s +[`gh-pages`](https://github.com/wwfxuk/tk-katana/tree/gh-pages) + +- `Dockerfile`: + - Moved `Gem*` from `/app` to `/usr/local/src`. + - Keeps `/app` clean for `docker run`. + - Added default `ENTRYPOINT` to run `build_docs.sh`. + - Added `tk-core` as Sphinx `conf.py` requires it. + - Optimised `yum install` packages, Ruby curl/untar. + - Using `nproc`, replacing hard-coded jobs count. + +- `preview_docs.sh`: + - Support `-v` instead of `--mount` for `docker run` when using older + `docker` version shipped with CentOS 7. + - Will not re-build Docker image if `tk-doc-generator` image exists. + - Force rebuild using `preview_docs.sh --rebuild`. + - Can now be run anywhere, including within `tk-doc-generator` + +- `build_docs.sh`: + - Fixed configurations using absolute paths. + - Fixed inaccurate echo statements +- `Gemfile*`: Using WWFX `just-the-docs` with + [new external links test](https://github.com/shotgunsoftware/just-the-docs/pull/9) + +# v1.x.x (v1.0.0+wwfx.0.1.0) + +Added this change log as well as minor fixes: + +- `.travis.yml`: Changed to WWFX UK `DOC_URL` +- `Dockerfile`: Fixed `yum clean all` from being part of the `yum install` arguments +- `build_docs.sh`: Fixed permissions of generated docs from just `root` only +- `travis-generate-docs.py`: Fallback to `DOC_*` before using dummy URL. + +# v1.0.0 + +Initial release from Shotgun, nothing mentioned by Shotgun. \ No newline at end of file diff --git a/preview_docs.sh b/preview_docs.sh index 453ec2b..60a4968 100755 --- a/preview_docs.sh +++ b/preview_docs.sh @@ -7,12 +7,85 @@ # this software in either electronic or hard copy form. # -# Builds a local preview of the documentation in the parent repository that is -# including tk-doc-generator as a submodule. +set -eu -o pipefail + +ARG1="${1:-}" +case "${ARG1}" in + -h|--help) + cat << EOF +Usage: preview_docs.sh [--rebuild|-h] + +Builds a local preview of the documentation in the parent repository that +includes tk-doc-generator as a sub-module. + +Options: + --rebuild Force re-build docker image. By default (for speed), + it will only build a new Docker image if one doesn't exist. + -h, --help Print this help and exit 0. + +Environment variables: + IMAGE_TAG: + Tag for the image Docker builds. + Defaults "tk-doc-generator" if not set + + MOUNT_TO: + Destination directory which the parent directory this script, + the folder above tk-doc-generator, is mounted to in the container. + Defaults "/app" if not set + +Examples: + Typically, this command is run in the parent repository that includes + tk-doc-generator as a sub-module. + + To force an image rebuild by running before creating documentation: + + tk-doc-generator/preview_docs.sh --rebuild + + Some environment variables will also influence this script's behaviour. + For example, you can run the script like this to build/run the docker image + tagged "my-doc-generator", you can run: + + IMAGE_TAG="my-doc-generator" tk-doc-generator/preview_docs.sh + +EOF + exit + ;; +esac + +# Setup useful directory variables for current script's folder/parent folder THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -docker build ${THIS_DIR}/../tk-doc-generator -t tk-doc-generator && \ -docker run --mount type=bind,source="${THIS_DIR}/..",target=/app tk-doc-generator \ -/app/tk-doc-generator/scripts/build_docs.sh --url=http://localhost --url-path=${THIS_DIR}/../_build --source=/app/docs --output=/app/_build && \ -echo "Documentation built in ${THIS_DIR}/../_build/index.html" + +# Build if image does not exist OR first argument is --rebuild +IMAGE_TAG="${IMAGE_TAG:-tk-doc-generator}" +if [ -z "$(docker images --quiet ${IMAGE_TAG})" -o "${ARG1}" == "--rebuild" ] +then + docker build --tag "${IMAGE_TAG}" ${THIS_DIR} +else + echo "'${IMAGE_TAG}' docker image already built" + echo "To force a re-build, try running this script with --rebuild flag" + echo +fi + + +# Setup mount/volume flags depending on which one current docker supports +MOUNT_TO="${MOUNT_TO:-/app}" +if ( docker run --help | grep --quiet -- --mount ) +then + MOUNT_FLAGS='--mount '"type=bind,source=$(pwd),target=${MOUNT_TO}" +else + MOUNT_FLAGS='-v '"$(pwd):${MOUNT_TO}" +fi + + +# Finally, perform the document generation in container +docker run --rm \ + ${MOUNT_FLAGS} \ + ${IMAGE_TAG} \ + --url="http://localhost" \ + --url-path="$(pwd)/_build" \ + --source="${MOUNT_TO}/docs" \ + --output="${MOUNT_TO}/_build" + +echo "Documentation built. Open in web-browser: $(pwd)/_build/index.html" diff --git a/scripts/build_docs.sh b/scripts/build_docs.sh index b7f0adf..d5075fb 100755 --- a/scripts/build_docs.sh +++ b/scripts/build_docs.sh @@ -53,8 +53,11 @@ echo "---------------------------------------------------" THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -TMP_FOLDER=${THIS_DIR}/../_doc_generator_tmp -TMP_BUILD_FOLDER=${TMP_FOLDER}/markdown_src +TK_DOC_GEN_SRC="$(readlink -m ${THIS_DIR}/..)" # contains Gemfile, jekyll, etc +OUR_REPO_ROOT="$(readlink -m ${SOURCE}/..)" # user's project root folder + +TMP_FOLDER="${TK_DOC_GEN_SRC}/_doc_generator_tmp" +TMP_BUILD_FOLDER="${TMP_FOLDER}/markdown_src" echo "" echo "Intermediate files will be written to '${TMP_FOLDER}'." @@ -67,35 +70,32 @@ echo "Cleaning out final build location '${OUTPUT}'..." rm -rf ${OUTPUT} echo "Creating build location '${TMP_BUILD_FOLDER}'..." -mkdir -p ${TMP_BUILD_FOLDER} +mkdir -p ${TMP_BUILD_FOLDER}/_plugins -echo "Copying source files into '${TMP_FOLDER}'..." +echo "Copying source files into '${TMP_BUILD_FOLDER}'..." cp -r ${SOURCE}/* ${TMP_BUILD_FOLDER} -echo "Copying plugins into '${TMP_FOLDER}/_plugins'..." -mkdir -p ${TMP_BUILD_FOLDER}/_plugins -cp -nr ${THIS_DIR}/../jekyll/_plugins/* ${TMP_BUILD_FOLDER}/_plugins +echo "Copying plugins into '${TMP_BUILD_FOLDER}/_plugins'..." +cp -nr ${TK_DOC_GEN_SRC}/jekyll/_plugins/* ${TMP_BUILD_FOLDER}/_plugins echo "Running Sphinx RST -> Markdown build process..." python ${THIS_DIR}/build_sphinx.py ${TMP_BUILD_FOLDER} echo "Running Jekyll to generate html from markdown..." +CONFIGS="${TK_DOC_GEN_SRC}/jekyll/_config.yml" # see if an external override config file exists -OVERRIDE_CONFIG=${THIS_DIR}/../../jekyll_config.yml - -if [ -e "$OVERRIDE_CONFIG" ]; then +OVERRIDE_CONFIG=${OUR_REPO_ROOT}/jekyll_config.yml +if [ -e "${OVERRIDE_CONFIG}" ] +then echo "using override config from ${OVERRIDE_CONFIG}..." - BUNDLE_GEMFILE=${THIS_DIR}/../Gemfile JEKYLL_ENV=production \ - bundle exec jekyll build \ - --baseurl "${URLPATH}" --config ${THIS_DIR}/../jekyll/_config.yml,${OVERRIDE_CONFIG} \ - --source "${TMP_BUILD_FOLDER}" --destination "${OUTPUT}" -else - BUNDLE_GEMFILE=${THIS_DIR}/../Gemfile JEKYLL_ENV=production \ + CONFIGS="${CONFIGS},${OVERRIDE_CONFIG}" +fi + +BUNDLE_GEMFILE=${TK_DOC_GEN_SRC}/Gemfile JEKYLL_ENV=production \ bundle exec jekyll build \ - --baseurl "${URLPATH}" --config ${THIS_DIR}/../jekyll/_config.yml \ + --baseurl "${URLPATH}" --config "${CONFIGS}" \ --source "${TMP_BUILD_FOLDER}" --destination "${OUTPUT}" -fi echo "------------------------------------------------------" echo "Build completed." diff --git a/scripts/get-tk-core-packages.sh b/scripts/get-tk-core-packages.sh new file mode 100755 index 0000000..b966142 --- /dev/null +++ b/scripts/get-tk-core-packages.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Copyright 2019 Autodesk, Inc. All rights reserved. +# +# Use of this software is subject to the terms of the Autodesk license agreement +# provided at the time of installation or download, or which otherwise accompanies +# this software in either electronic or hard copy form. +# + +set -eu -o pipefail + +usage() { +cat << EOF +Usage: get-tk-core-packages.sh [-h] FOLDER + +Downloads the latest (tagged) version of tk-core into a given FOLDER. +If FOLDER does not exist, it will try create it. + +Options: + -h, --help Print this help and exit 0 + +Example: + To download the latest sgtk, tank and tank_vendor package folders into + /usr/local/lib64/python, run: + + get-tk-core-packages.sh /usr/local/lib64/python + + This should result in a sgtk, tank and tank_vendor directly under + /usr/local/lib64/python. +EOF +} + +if [ $# -ne 1 ] +then + echo 'ERROR: Please provide a folder to download/install into!' + echo + usage + exit 1 +fi + +case $1 in + -h|--help) + usage + exit + ;; +esac + +# Try create folder if it does not currently exist +FOLDER="$1" +[ -d "${FOLDER}" ] || mkdir -vp "${FOLDER}" + +curl_untar() { + # GitHub API to get all tags (seems to be latest first/at top of file) + TAGS_FILE="$(mktemp)" + curl -s "https://api.github.com/repos/shotgunsoftware/tk-core/tags" \ + | tee ${TAGS_FILE} + + # Grab first version (tag name), extract sgtk, tank and tank_vendor packages + VERSION=$(grep --color=never -m 1 -o -P '(?<="name": ")[^"]+' ${TAGS_FILE}) + + echo "Downloading and extracting tk-core ${VERSION} into: ${FOLDER}" + curl -Ls https://github.com/shotgunsoftware/tk-core/archive/${VERSION}.tar.gz \ + | tar --directory "${FOLDER}" \ + --extract --gzip --file=- \ + --exclude='*tests*' --wildcards --wildcards-match-slash \ + --strip-components=2 '*python*' +} + +temp_repo() { + # Clone into a temporary directory + TEMP_REPO="$(mktemp -d)" + pushd "${TEMP_REPO}" + git clone --no-checkout https://github.com/shotgunsoftware/tk-core.git . + + # Checkout latest version, grab the files/folders under python folder + VERSION="$(git for-each-ref --sort=taggerdate --format '%(tag)' | tail -1)" + echo "Cloned and extracting tk-core ${VERSION} into: ${FOLDER}" + git checkout "${VERSION}" + mv python/* "${FOLDER}" + popd +} + +# Try faster curl_untar first before falling back to temporary clone method +curl_untar || temp_repo +ls -lh "${FOLDER}" \ No newline at end of file diff --git a/travis-generate-docs.py b/travis-generate-docs.py index f7869e3..ce40006 100755 --- a/travis-generate-docs.py +++ b/travis-generate-docs.py @@ -124,27 +124,30 @@ def main(): current_branch = os.environ.get("TRAVIS_BRANCH") inside_pr = os.environ.get("TRAVIS_PULL_REQUEST") != "false" - # first figure out i we are on master or in a PR. + # first figure out if we are on master or in a PR. if current_branch != "master" or inside_pr: - # we are in a PR. - log.info("Inside a pull request.") + log.info("Inside a pull request (or not master branch).") - # see if we have access to an AWS bucket - if "S3_BUCKET" in os.environ and "S3_WEB_URL" in os.environ: + # See if we have access to an AWS bucket + s3_bucket = os.getenv("S3_BUCKET", default=None) + target_url = os.getenv("S3_WEB_URL", default=None) + if s3_bucket and target_url: log.info("Detected AWS S3 bucket for preview workflow.") - - s3_bucket = os.environ["S3_BUCKET"] - target_url = os.environ["S3_WEB_URL"] - target_url_path = "/tk-doc-generator/{commit}".format( - commit=os.environ["TRAVIS_COMMIT"] - ) - + target_url_path = "/tk-doc-generator/{env[TRAVIS_COMMIT]}" + target_url_path = target_url_path.format(env=os.environ) else: log.warning("No S3_BUCKET and S3_WEB_URL detected in environment. " "No S3 preview will be generated") - s3_bucket = None - # enter dummy paths so we can at least build - # the docs to check for errors + + # Then try use DOC_* for target + target_url = os.getenv("DOC_URL", default=None) + target_url_path = os.getenv("DOC_PATH", default=None) + if target_url and target_url_path: + log.info("Using DOC_URL/PATH.") + else: + # Fall back to using root directly on dummy domain url + log.warning("Using dummy paths so we can at least build the docs " + "to check for errors") target_url = "https://dummy.url.com" target_url_path = "/" @@ -160,12 +163,14 @@ def main(): ) execute_external_command(doc_command) - if s3_bucket: + aws_access_key = os.getenv("AWS_S3_ACCESS_KEY", default=None) + aws_access_token = os.getenv("AWS_S3_ACCESS_TOKEN", default=None) + if s3_bucket and aws_access_key and aws_access_token: log.info("Uploading build result to S3...") s3_client = boto3.client( "s3", - aws_access_key_id=os.environ["AWS_S3_ACCESS_KEY"], - aws_secret_access_key=os.environ["AWS_S3_ACCESS_TOKEN"] + aws_access_key=aws_access_key, + aws_access_token=aws_access_token ) # note: skip the first slash when uploading to S3