diff --git a/.gitignore b/.gitignore index a4a2fd5..45f8e10 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea .ipynb_checkpoints */~* +*.swp ~* diff --git a/tutorials/tut02/README.md b/tutorials/tut02/README.md index 180188e..a00d80b 100644 --- a/tutorials/tut02/README.md +++ b/tutorials/tut02/README.md @@ -14,10 +14,9 @@ I will read data from the MIDAS-Open dataset and aggregate all measurements (fro An example temperature file for a single station and year (in the MIDAS-Open data set) can be found at: - /badc/ukmo-midas-open/data/uk-daily-temperature-obs/dataset-version-201901/ - devon/01359_cheldon-barton/qc-version-1/ - midas-open_uk-daily-temperature-obs_dv- - 201901_devon_01359_cheldon-barton_qcv-1_1977.csv +```bash +/badc/ukmo-midas-open/data/uk-daily-temperature-obs/dataset-version-201901/devon/01359_cheldon-barton/qc-version-1/midas-open_uk-daily-temperature-obs_dv-201901_devon_01359_cheldon-barton_qcv-1_1977.csv +``` This workflow will generate a graph of a time series of maximum temperature data that looks something like: @@ -37,11 +36,9 @@ When they have all completed then the **(3)** finalisation task can be executed. The actual tasks are: - 1. **Initialisation**: Clone the repository to get the extraction scripts; run the first script - to generate a list of UK counties; write those to a text file. + 1. **Initialisation**: Clone the repository to get the extraction scripts; run the first script to generate a list of UK counties; write those to a text file. - 2. **Batch**: For each county: calculate a 2000-2017 time series of the annual maximum - temperature across all stations. + 2. **Batch**: For each county: calculate a 2000-2017 time series of the annual maximum temperature across all stations. 3. **Finalisation**: Read in all the time series files and plot them on a line graph to a PNG file. @@ -54,15 +51,15 @@ It may help to sketch out the process in a simple diagram, as shown here... After completing this tutorial I will be able to: * break down a multi-step workflow into independent tasks - * configure a basic Rose/Cylc job that includes a multi-step workflow + * configure a basic Cylc job that includes a multi-step workflow * run the workflow on the `cylc` server on JASMIN * interact with the Cylc graphical user interface ### JASMIN resources * JASMIN account with SSH public key uploaded and `jasmin-login` privilege - * `login` server: `login.jasmin.ac.uk` - * Cylc server: `cylc.jasmin.ac.uk` + * login servers: `login.jasmin.ac.uk` + * Cylc server: `cylc2.jasmin.ac.uk` * LOTUS batch processing cluster * GWS (read/write): `/gws/pw/j07/workshop` * `$HOME` directory @@ -77,36 +74,36 @@ After completing this tutorial I will be able to: This is the outline of the overall task. The recommended way of doing each step is covered in the "Walkthrough" below. - 1. The starting point is on the JASMIN `login` server (see [exercise 01](../ex01)) - 1. SSH to the Rose & Cylc server (with the `-X` flag to forward X-windows) - 1. In this example you are given the building blocks to construct a "suite file" for use with Rose & Cylc - 1. Wrap the scripts in a Rose suite by copying the example suite to a new directory called `my-suite` and modifying it - 1. Run the Rose suite - 1. If the suite partially runs and leaves log/working directories in place you can clean these up and run it again - 1. If you need to stop the suite then you can instruct Cylc to stop it + 1. The starting point is on a JASMIN `login` server (see [exercise 01](../ex01)) + 1. SSH to the Cylc server (with the `-X` flag to forward X-windows) + 1. In this example you are given the building blocks to construct a "workflow file" for use with Cylc + 1. Wrap the scripts in a Cylc workflow by copying the example workflow to a new directory called `my-workflow` and modifying it + 1. Run the Cylc workflow + 1. If the workflow partially runs and leaves log/working directories in place you can clean these up and run it again + 1. If you need to stop the workflow then you can instruct Cylc to stop it ### Review / alternative approaches / best practice Alternative approaches and good practice might include: - * Manage the process yourself (without Rose and Cylc)? + * Manage the process yourself (without Cylc)? * Set the `$PATH` environment variable in your `~/.bash_profile` * Write your outputs somewhere else * *Have any files been accidentally left on the system?* (e.g. in: `/tmp/` etc) - * Tidy up your run suite directory (i.e. logs and task directories) - * View the workflow graph of the suite - * Understand different modes of stopping a running suite + * Tidy up your run workflow directory (i.e. logs and task directories) + * View the workflow graph of the workflow + * Understand different modes of stopping a running workflow ### Walkthrough 1. The starting point is on the JASMIN `login` server (see [exercise 01](../ex01)) -2. SSH to the Rose & Cylc server (with the `-X` flag to forward X-windows) +2. SSH to the Cylc server (with the `-X` flag to forward X-windows) ``` - ssh -X cylc + ssh -X cylc2 ``` -3. In this example you are given the building blocks to construct a "suite file" for use with Rose & Cylc +3. In this example you are given the building blocks to construct a "flow file" for use with Cylc * Script 1: `create-counties-file.py` 1. Context: Python 3 @@ -130,86 +127,98 @@ Alternative approaches and good practice might include: * On JASMIN: `/gws/pw/j07/workshop/tutorials/tut02/code/` * On github: https://github.com/cedadev/jasmin-workshop/tree/master/tutorials/tut02/code -4. Wrap the scripts in a Rose suite by copying the example suite to a new directory called - `my-suite` and modifying it. +4. Wrap the scripts in a Cylc workflow by copying the example workflow to a new directory called + `my-workflow` and modifying it. - * The example suite is available at: + * The example workflow is available at: - /gws/pw/j07/workshop/tutorials/tut02/example-suite + /gws/pw/j07/workshop/tutorials/tut02/example-workflow - * Go to the directory where you have copied the suite. + * Go to the directory where you have copied the workflow. - * You can run the example suite to view how it works, with: + ``` + cd my-workflow + ``` + + * Update the SLURM configuration file in `flow.cylc` to use your own account. + + ``` + # In flow.cylc + # Change: + # --account=jasmin-workshop + # To: + # --account= + ``` + + * You can run the copied example workflow to view how it works, with: ``` - # Add the location of the rose/cylc executables to $PATH + # Add the location of the cylc executables to $PATH export PATH=/apps/jasmin/metomi/bin:$PATH - rose suite-run + cylc validate . + cylc install . + cylc play my-workflow ``` - * All of the scripts operate on input/output data in the relative directory: `./outputs`. It - therefore makes sense to copy the Python scripts to the main suite "run directory" and - ensure that each task runs from that directory. The suite run directory is specified by - the Cylc environment variable: `$CYLC_SUITE_RUN_DIR`. + This is the same as running `cylc vip` + + * All of the scripts operate on input/output data in the relative directory: `./outputs`. It therefore makes sense to copy the Python scripts to the main workflow "run directory" and ensure that each task runs from that directory. The workflow run directory is specified by the Cylc environment variable: `$CYLC_WORKFLOW_RUN_DIR`. - This variable is set for you by Cylc; a typical value would be - `/home/users/$USER/cylc-run/my-suite`. Note that the current working directory for the - individual steps is different for each step (for example - `/home/users/$USER/cylc-run/my-suite/work/1/initialise` for the `initialise` step). For the - workflow below, it is convenient for all steps to be run in the same directory, so the commands - for each step will include changing directory to `$CYLC_SUITE_RUN_DIR` before running the - relevant script. + This variable is set for you by Cylc; a typical value would be `/home/users/$USER/cylc-run/my-workflow/run1`. Note that the current working directory for the individual steps is different for each step (for example `/home/users/$USER/cylc-run/my-workflow/run1/work/1/initialise` for the `initialise` step). For the workflow below, it is convenient for all steps to be run in the same directory, so the commands for each step will include changing directory to `$CYLC_WORKFLOW_RUN_DIR` before running the relevant script. + Since the example workflow currently only uses `echo` placeholders, it does not generate any outputs yet. - * Edit the `suite.rc` file as follows: - * In the `[[runtime]]` section of the suite file, modify each of the 4 processing steps as follows: + * To make the workflow do something useful with the Python scripts, you would normally edit the `flow.cylc` file as follows: + * In the `[[runtime]]` section of the workflow file, modify each of the 4 processing steps as follows: * `[[initialise]]` - 1. Delete any existing `jasmin-workshop` sub-directory (in case the suite has - been run previously) + 1. Delete any existing `jasmin-workshop` sub-directory (in case the workflow has been run previously) 2. Clone the GitHub repository: https://github.com/cedadev/jasmin-workshop - 3. Copy the files in the sub-directory `jasmin-workshop/tutorials/tut02/code/` to the - suite run directory at: `$CYLC_SUITE_RUN_DIR` + 3. Copy the files in the sub-directory `jasmin-workshop/tutorials/tut02/code/` to the workflow run directory at: `$CYLC_WORKFLOW_RUN_DIR`. * `[[step1]]` - 1. Delete any existing `outputs` sub-directory (in case the suite has been run - previously) + 1. Delete any existing `outputs` sub-directory (in case the workflow has been run previously) 2. Activate the standard JASMIN Python 3 environment. - 3. Change directory to the `$CYLC_SUITE_RUN_DIR` + 3. Change directory to the `$CYLC_WORKFLOW_RUN_DIR` 4. Run the script * `[[batch]]` 1. Activate the standard JASMIN Python 3 environment. - 2. Change directory to the `$CYLC_SUITE_RUN_DIR` + 2. Change directory to the `$CYLC_WORKFLOW_RUN_DIR` 3. Run the script for each value of the `counter`: * The `counter` variable is accessible by the environment variable: `CYLC_TASK_PARAM_counter` * `[[final]]` 1. Activate the standard JASMIN Python 3 environment. - 2. Change directory to the `$CYLC_SUITE_RUN_DIR` + 2. Change directory to the `$CYLC_WORKFLOW_RUN_DIR` 3. Run the script -5. Run the Rose suite with the command: + * **Instead of manually modifying the `my-workflow` configuration right now**, the repository already includes a fully configured workflow in the `workshop-workflow` directory that implements all of the above steps. + +5. Run the fully configured `workshop-workflow`: ``` + # Add the location of the cylc executables to $PATH export PATH=/apps/jasmin/metomi/bin:$PATH - rose suite-run + cd workshop-workflow/ + # Remember to update the account in flow.cylc to as you did before. + cylc vip . ``` - * **NOTE:** It will take a couple of minutes to start up and then a GUI should appear that - shows the workflow in action. + * **NOTE:** It will take a couple of minutes to start up. To view the workflow in action, open the GUI with: -6. If the suite partially runs and leaves log/working directories in place you can clean these up and run it again with: +6. If the workflow partially runs and leaves log/working directories in place you can clean these up and run it again with: ``` - rose suite-run --new + cylc clean workshop-workflow + cylc vip . ``` -7. If you need to stop the suite then you can instruct Cylc to stop it with: +7. If you need to stop the workflow then you can instruct Cylc to stop it with: ``` - cylc stop '' - # Where is the name of the suite directory + cylc stop '' + # Where is the name of the workflow directory ``` ### What should the Cylc GUI look like? @@ -221,15 +230,25 @@ Note that in this example the 4 tasks have been renamed to: * `process` * `plot` -The full example suite is available at: +The full example workflow is available at: - /gws/pw/j07/workshop/tutorials/tut02/workshop-suite + /gws/pw/j07/workshop/tutorials/tut02/workshop-workflow And on github at: - https://github.com/cedadev/jasmin-workshop/tree/master/tutorials/tut02/workshop-suite + https://github.com/cedadev/jasmin-workshop/tree/master/tutorials/tut02/workshop-workflow + -The Cylc GUI opens when you start running a suite (as long as you have used the `-X` flag when SSHing to the server). +#### Cylc TUI + +The Cylc TUI can be opened by running `cylc tui`, and is an easily accessible alternative to the GUI that runs in the terminal and provides similar levels of functionality to the GUI and can act as a quick way to check on the status of your workflow. + +![Cylc TUI Screenshot 1](./images/tui1.png) + + +#### Cylc GUI + +The Cylc GUI can be opened by running `cylc gui` (as long as you have used the `-X` flag when SSHing to the server), it will open up in a browser. ![Cylc GUI Screenshot 1](./images/gui1.png) @@ -241,35 +260,48 @@ The graph view shows each of the tasks and the dependency graph that connects th ![Cylc GUI Screenshot 3](./images/gui3.png) -Failed tasks are clearly indicated in red. +Failed tasks are clearly indicated in red. To emulate this failure, you can edit the `flow.cylc` file and change the `command` for the `clone_repo` task to something that will fail, for example: + +``` + [[clone_repo]] + script = """rm -fr workshop-repo +git clone https://github.com/cedadev/jasmin-workshop workshop-repo +cp workshop-repo/tutorials/tut02/code/* ${CYLC_WORKFLOW_RUN_DIR}/ +rm -fr workshop-repo +exit 1""" # <-- Added exit 1 to emulate failure +``` ![Cylc GUI Screenshot 4](./images/gui4.png) -Right-click on a failed task in order to view more information. +Click on the menu button at the top for more options. ![Cylc GUI Screenshot 5](./images/gui5.png) -Each of the log files can be viewed within the GUI. +Each of the tasks has a log file that can be viewed within the GUI. By default, opening the logs will show the log for the entire workflow. ![Cylc GUI Screenshot 6](./images/gui6.png) +You can opt to view the specific job logs by selecting the `JOB` tab instead of the default `WORKFLOW` tab, then, enter the `/` in the search box to get the logs corresponding to that task, in this case `1/clone_repo`. + +![Cylc GUI Screenshot 7](./images/gui7.png) + ### Details of alternative approaches and best practice -> 1. Manage the process yourself (without Rose and Cylc)? +> 1. Manage the process yourself (without Cylc)? * Pros: - * You don't need to learn/configure Rose and Cylc + * You don't need to learn/configure Cylc * Cons: * You have to check the dependency tree yourself: * You need to check whether all tasks have run in a given stage before progressing to the next stage. - * With Rose and Cylc you can configure complex rules for responding to failures and retrying tasks. + * With Cylc you can configure complex rules for responding to failures and retrying tasks. > 2. Set the `$PATH` environment variable in your `~/.bash_profile` - * In order to find the Rose/Cylc variables we needed to change the `$PATH` as follows: + * In order to find the Cylc variables we needed to change the `$PATH` as follows: ``` - # Add the location of the rose/cylc executables to $PATH + # Add the location of the cylc executables to $PATH export PATH=/apps/jasmin/metomi/bin:$PATH ``` @@ -277,8 +309,8 @@ Each of the log files can be viewed within the GUI. automatically when you login to the cylc server: ``` - if [[ $HOSTNAME = "cylc1.jasmin.ac.uk" ]]; then - # NOTE: "cylc" is an alias to the "cylc1" server + if [[ $HOSTNAME = "cylc2.jasmin.ac.uk" ]]; then + # NOTE: "cylc" is an alias to the "cylc2" server export PATH=/apps/jasmin/metomi/bin:$PATH fi ``` @@ -288,74 +320,72 @@ Each of the log files can be viewed within the GUI. * You might write outputs directly to a Group Workspace * You might write outputs to the default working directory for a task: * These are symbolically linked to the JASMIN `/work/scratch-pw/$USER` area. - * See more details in the Rose documentation: - https://metomi.github.io/rose/doc/html/tutorial/cylc/runtime/introduction.html#where-do-all-the-files-go + * See more details in the Cylc documentation: + https://cylc.github.io/cylc-doc/stable/html/workflow-design-guide/general-principles.html#self-contained-workflows > 4. *Have any files been accidentally left on the system?* (e.g. in: `/tmp/` etc) - * Running a Rose suite will copy your suite to a "run suite directory" under: + * Running a Cylc workflow will copy your workflow to a "run workflow directory" under: - $HOME/cylc-run// + $HOME/cylc-run//runN/ * This directory includes various files, directories and symbolic links related to your job. Please check that you are not writing big files to that the directory and monitor the size of the outputs. -> 5. Tidy up your run suite directory (i.e. logs and task directories) +> 5. Tidy up your run workflow directory (i.e. logs and task directories) - * You can tell Rose to tidy up (clear out) any logs and task directories by using the command: + * You can tell Cylc to tidy up (clear out) any logs and task directories by using the command: ``` - rose suite-clean + cylc clean ``` -> 6. View the workflow graph of the suite +> 6. View the workflow graph of the workflow - * View the workflow graph of the suite: - * To view the workflow graph of your suite without running it, use: + * View the workflow graph of the workflow: + * To view the workflow graph of your workflow without running it in the browser, use: ``` - rose suite-run -i - cylc graph '' - # Where is the name of the suite directory + cylc graph '' + # Where is the name of the workflow directory ``` - * NOTE: the `-i` option means "install only" - so this will not run the suite. - ![Image of workflow graph](./images/gui7.png) + * NOTE: this will only open and visualise the workflow graph in a browser and will not run the workflow. + ![Image of workflow graph](./images/gui8.png) -> 7. Understand different modes of stopping a running suite +> 7. Understand different modes of stopping a running workflow - * Understand different modes of stopping a running suite: - * If you need to stop a suite that is running you can use: + * Understand different modes of stopping a running workflow: + * If you need to stop a workflow that is running you can use: ``` - cylc stop '' - # Where is the name of the suite directory + cylc stop '' + # Where is the name of the workflow directory ``` - * The `cylc stop` command may not stop the suite immediately - because it will wait + * The `cylc stop` command may not stop the workflow immediately - because it will wait for submitted and running tasks to complete. - * To kill the submitted and running tasks before stopping the suite, use: + * To kill the submitted and running tasks before stopping the workflow, use: ``` - cylc stop --kill '' + cylc stop --kill '' ``` - * To stop the suite regardless of submitted and running tasks, use: + * To stop the workflow regardless of submitted and running tasks, use: ``` - cylc stop --now '' + cylc stop --now '' ``` ### Review and further info This tutorial demonstrates how to: - * Use the Rose and Cylc workflow management tools. - * Construct a Rose suite involving a multi-step workflow. - * Configure a Rose suite to work with the LOTUS batch cluster on JASMIN. - * Run a Rose suite and monitor its progress using the Cylc GUI. - -Rose and Cylc are very versatile tools. We recommend that you study the documentation at: - * Rose: https://metomi.github.io/rose/doc/html/ - * Cylc: https://cylc.github.io/doc/built-sphinx/ + * Use the Cylc workflow management tool. + * Construct a Cylc workflow involving a multi-step workflow. + * Configure a Cylc workflow to work with the LOTUS batch cluster on JASMIN. + * Run a Cylc workflow and monitor its progress using the Cylc GUI. + +Cylc is a very versatile tool. We recommend that you study the documentation at: + * Cylc: https://cylc.github.io/cylc-doc/stable/html/ diff --git a/tutorials/tut02/example-suite/README.md b/tutorials/tut02/example-suite/README.md deleted file mode 100644 index ca7ebb8..0000000 --- a/tutorials/tut02/example-suite/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Running the suite - -## To run the suite - -`rose suite-run` - -## To stop the suite (if failed/still running) - -`cylc stop 'example-suite'` - -## To clean out the suite working directory - -`rose suite-clean --yes` - -This can be required before re-running. - -Alternatively, you can force clean-up and re-run the suite with one command: - -`rose suite-run --new` - -## Working directory - -When Cylc runs the suite it has a working directory where the suite is run. -This is referenced in the `suite.rc` file with the environment variable -`$CYLC_SUITE_RUN_DIR`. - -By default this will be under: - -``` -$HOME/cylc-run// -``` - -### Stopping a suite that is running - -Note: In our example suite, named 'example-suite', you would change the value of '' to -'example-suite' in the following commands. - -If you need to stop the suite you can use: - -``` -cylc stop '' -``` - -The `cylc stop` command may not stop the suite immediately - because it will wait for submitted -and running tasks to complete. - -To kill the submitted and running tasks before stopping the suite, use: - -``` -cylc stop --kill '' -``` - -To stop the suite regardless of submitted and running tasks, use: - -``` -cylc stop --now '' -``` - -### Working directory for the example suite - -In the case of the `example-suite` the working directory will be: - -``` -$HOME/cylc-run/example-suite/ -``` - -The example suite uses this as the common location for all scripts and the outputs -directory to be located in. - -### Viewing the workflow graph of the suite - -To view the workflow graph of your suite _without_ running it, use: - -``` -rose suite-run -i -cylc graph example-suite -``` - -NOTE: the `-i` option means "install only" - so this will not run the suite. diff --git a/tutorials/tut02/example-workflow/README.md b/tutorials/tut02/example-workflow/README.md new file mode 100644 index 0000000..9a2915d --- /dev/null +++ b/tutorials/tut02/example-workflow/README.md @@ -0,0 +1,109 @@ +# Running the workflow + +## To run the workflow + +`cylc validate .` +`cylc install .` +`cylc play example-workflow` + +## To stop the workflow (if failed/still running) + +`cylc stop example-workflow` + +## To clean out the workflow working directory + +`cylc clean example-workflow` + +This can be required before re-running. + +## Working directory + +When Cylc runs the workflow it has a working directory where the workflow is run. +This is referenced in the `flow.cylc` file with the environment variable +`$CYLC_WORKFLOW_RUN_DIR`. + +By default this will be under: + +``` +$HOME/cylc-run//runN/ +``` + +### Stopping a workflow that is running + +Note: In our example workflow, named 'example-workflow', you would change the value of '' to +'example-workflow' in the following commands. + +If you need to stop the workflow you can use: + +``` +cylc stop '' +``` + +The `cylc stop` command may not stop the workflow immediately - because it will wait for submitted +and running tasks to complete. + +To kill the submitted and running tasks before stopping the workflow, use: + +``` +cylc stop --kill '' +``` + +To stop the workflow regardless of submitted and running tasks (shut down immediately, leave active tasks alone), use: + +``` +cylc stop --now '' +``` + +### Working directory for the example workflow + +In the case of the `example-workflow` the working directory will be: + +``` +$HOME/cylc-run/example-workflow/runN/ +``` + +The example workflow uses this as the common location for all scripts and the outputs +directory to be located in. + +### Viewing the workflow graph of the workflow + +To view the workflow graph of your workflow _without_ running it, use: + +``` +cylc install +cylc graph example-workflow +``` + +NOTE: `cylc install` only installs the workflow. This will not run the workflow until `cylc play` is run. + + +### Viewing available platform options on the system + +``` +cylc config --platforms +``` + +This will list the options you have available that you can use in your `flow.cylc` file - such as running on the local system, or on lotus, e.g.: + +``` +$ cylc config --platforms +[platforms] + [[localhost]] + install target = localhost + copyable environment variables = FCM_VERSION + submission polling intervals = PT30M + execution polling intervals = PT30M + execution time limit polling intervals = PT5M, PT10M + clean job submission environment = True + [[lotus]] + install target = localhost + copyable environment variables = FCM_VERSION + submission polling intervals = PT30M + execution polling intervals = PT30M + execution time limit polling intervals = PT5M, PT10M + clean job submission environment = True + hosts = localhost + job runner = slurm + [[[meta]]] + description = LOTUS Slurm job +``` diff --git a/tutorials/tut02/example-suite/suite.rc b/tutorials/tut02/example-workflow/flow.cylc similarity index 70% rename from tutorials/tut02/example-suite/suite.rc rename to tutorials/tut02/example-workflow/flow.cylc index 19c12da..8247fe8 100644 --- a/tutorials/tut02/example-suite/suite.rc +++ b/tutorials/tut02/example-workflow/flow.cylc @@ -1,24 +1,23 @@ -[cylc] - UTC mode = True # Ignore DST - [[parameters]] - counter = 1..20 +[task parameters] + counter = 1..20 [scheduling] - [[dependencies]] - graph = """ + [[graph]] + R1 = """ initialise => step1 step1 => batch - batch => finish""" + batch => finish + """ [runtime] [[root]] # This is applied before every task pre-script="""set -eu""" - [[[job]]] - batch system = slurm - execution time limit = PT5M + execution time limit = PT5M + platform = lotus # See options via `cylc config --platforms`, e.g. `localhost`, `lotus` [[[directives]]] - --partition = workshop + --partition = debug + --qos = debug --account = workshop --time = 05:00 [[initialise]] diff --git a/tutorials/tut02/example-suite/rose-suite.conf b/tutorials/tut02/example-workflow/rose-suite.conf similarity index 100% rename from tutorials/tut02/example-suite/rose-suite.conf rename to tutorials/tut02/example-workflow/rose-suite.conf diff --git a/tutorials/tut02/example-suite/rose-suite.info b/tutorials/tut02/example-workflow/rose-suite.info similarity index 100% rename from tutorials/tut02/example-suite/rose-suite.info rename to tutorials/tut02/example-workflow/rose-suite.info diff --git a/tutorials/tut02/images/gui1.png b/tutorials/tut02/images/gui1.png index 2922b8a..5ee42e7 100644 Binary files a/tutorials/tut02/images/gui1.png and b/tutorials/tut02/images/gui1.png differ diff --git a/tutorials/tut02/images/gui2.png b/tutorials/tut02/images/gui2.png index 4394571..9909d9d 100644 Binary files a/tutorials/tut02/images/gui2.png and b/tutorials/tut02/images/gui2.png differ diff --git a/tutorials/tut02/images/gui3.png b/tutorials/tut02/images/gui3.png index 2263e9c..526a65b 100644 Binary files a/tutorials/tut02/images/gui3.png and b/tutorials/tut02/images/gui3.png differ diff --git a/tutorials/tut02/images/gui4.png b/tutorials/tut02/images/gui4.png index f2f248a..b0e3c71 100644 Binary files a/tutorials/tut02/images/gui4.png and b/tutorials/tut02/images/gui4.png differ diff --git a/tutorials/tut02/images/gui5.png b/tutorials/tut02/images/gui5.png index 6403026..66a9068 100644 Binary files a/tutorials/tut02/images/gui5.png and b/tutorials/tut02/images/gui5.png differ diff --git a/tutorials/tut02/images/gui6.png b/tutorials/tut02/images/gui6.png index fa01ef3..0843c50 100644 Binary files a/tutorials/tut02/images/gui6.png and b/tutorials/tut02/images/gui6.png differ diff --git a/tutorials/tut02/images/gui7.png b/tutorials/tut02/images/gui7.png index e0c8dd9..c20f25e 100644 Binary files a/tutorials/tut02/images/gui7.png and b/tutorials/tut02/images/gui7.png differ diff --git a/tutorials/tut02/images/gui8.png b/tutorials/tut02/images/gui8.png new file mode 100644 index 0000000..3d665a0 Binary files /dev/null and b/tutorials/tut02/images/gui8.png differ diff --git a/tutorials/tut02/images/tui1.png b/tutorials/tut02/images/tui1.png new file mode 100644 index 0000000..d68db3a Binary files /dev/null and b/tutorials/tut02/images/tui1.png differ diff --git a/tutorials/tut02/workshop-suite/README.md b/tutorials/tut02/workshop-suite/README.md deleted file mode 100644 index fc7bb3c..0000000 --- a/tutorials/tut02/workshop-suite/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Running the suite - -## To run the suite - -`rose suite-run` - -## To stop the suite (if failed/still running) - -`cylc stop 'workshop-suite'` - -## To clean out the suite working directory - -`rose suite-clean --yes` - -This can be required before re-running. - -Alternatively, you can force clean-up and re-run the suite with one command: - -`rose suite-run --new` - -## Working directory - -When Cylc runs the suite it has a working directory where the suite is run. -This is referenced in the `suite.rc` file with the environment variable -`$CYLC_SUITE_RUN_DIR`. - -By default this will be under: - -``` -$HOME/cylc-run// -``` - -### Working directory for Workshop example suite - -In the case of the `workshop-suite` the working directory will be: - -``` -$HOME/cylc-run/workshop-suite/ -``` - -The workshop suite uses this as the common location for all scripts and the outputs -directory to be located in. - -The final output graph is written to: - -``` -$HOME/cylc-run/workshop-suite/outputs/annual-max-temp-time-series.png -``` - -### View the final time series graph of maximum temperatures - -You can view the output graph with this command: - -``` -display $HOME/cylc-run/workshop-suite/outputs/annual-max-temp-time-series.png -``` - -## Additional Rose & Cylc commands - -### Stopping a suite that is running - -Note: In our example suite, named 'example-suite', you would change the value of '' to -'example-suite' in the following commands. - -If you need to stop the suite you can use: - -``` -cylc stop '' -``` - -The `cylc stop` command may not stop the suite immediately - because it will wait for submitted -and running tasks to complete. - -To kill the submitted and running tasks before stopping the suite, use: - -``` -cylc stop --kill '' -``` - -To stop the suite regardless of submitted and running tasks, use: - -``` -cylc stop --now '' -``` - -### Viewing the workflow graph of the suite - -To view the workflow graph of your suite _without_ running it, use: - -``` -rose suite-run -i -cylc graph example-suite -``` - -NOTE: the `-i` option means "install only" - so this will not run the suite. - - diff --git a/tutorials/tut02/workshop-workflow/README.md b/tutorials/tut02/workshop-workflow/README.md new file mode 100644 index 0000000..fb0f0fd --- /dev/null +++ b/tutorials/tut02/workshop-workflow/README.md @@ -0,0 +1,125 @@ +# Running the workflow + +## To run the workflow + +`cylc validate .` +`cylc install .` +`cylc play workshop-workflow` + +## To stop the workflow (if failed/still running) + +`cylc stop workshop-workflow` + +## To clean out the workflow working directory + +`cylc clean workshop-workflow` + +This can be required before re-running. + +## Working directory + +When Cylc runs the workflow it has a working directory where the workflow is run. +This is referenced in the `flow.cylc` file with the environment variable +`$CYLC_WORKFLOW_RUN_DIR`. + +By default this will be under: + +``` +$HOME/cylc-run//runN/ +``` + +### Working directory for Workshop example workflow + +In the case of the `workshop-workflow` the working directory will be: + +``` +$HOME/cylc-run/workshop-workflow/runN/ +``` + +The workshop workflow uses this as the common location for all scripts and the outputs +directory to be located in. + +The final output graph is written to: + +``` +$HOME/cylc-run/workshop-workflow/runN/outputs/annual-max-temp-time-series.png +``` + +### View the final time series graph of maximum temperatures + +You can view the output graph with this command (`display` command is only available on login-0? and sci-vm-0? servers, not the cylc2.jasmin.ac.uk): + +``` +display $HOME/cylc-run/workshop-workflow/runN/outputs/annual-max-temp-time-series.png +``` + +## Additional Cylc commands + +### Stopping a workflow that is running + +Note: In our example workflow, named 'workshop-workflow', you would change the value of '' to +'workshop-workflow' in the following commands. + +If you need to stop the workflow you can use: + +``` +cylc stop '' +``` + +The `cylc stop` command may not stop the workflow immediately - because it will wait for submitted +and running tasks to complete. + +To kill the submitted and running tasks before stopping the workflow, use: + +``` +cylc stop --kill '' +``` + +To stop the workflow regardless of submitted and running tasks (shut down immediately, leave active tasks alone), use: + +``` +cylc stop --now '' +``` + +### Viewing the workflow graph of the workflow + +To view the workflow graph of your workflow _without_ running it, use: + +``` +cylc install +cylc graph workshop-workflow +``` + +NOTE: `cylc install` only installs the workflow. This will not run the workflow until `cylc play` is run. + + +### Viewing available platform options on the system + +``` +cylc config --platforms +``` + +This will list the options you have available that you can use in your `flow.cylc` file - such as running on the local system, or on lotus, e.g.: + +``` +$ cylc config --platforms +[platforms] + [[localhost]] + install target = localhost + copyable environment variables = FCM_VERSION + submission polling intervals = PT30M + execution polling intervals = PT30M + execution time limit polling intervals = PT5M, PT10M + clean job submission environment = True + [[lotus]] + install target = localhost + copyable environment variables = FCM_VERSION + submission polling intervals = PT30M + execution polling intervals = PT30M + execution time limit polling intervals = PT5M, PT10M + clean job submission environment = True + hosts = localhost + job runner = slurm + [[[meta]]] + description = LOTUS Slurm job +``` diff --git a/tutorials/tut02/workshop-suite/suite.rc b/tutorials/tut02/workshop-workflow/flow.cylc similarity index 53% rename from tutorials/tut02/workshop-suite/suite.rc rename to tutorials/tut02/workshop-workflow/flow.cylc index 4c7d618..cffc871 100644 --- a/tutorials/tut02/workshop-suite/suite.rc +++ b/tutorials/tut02/workshop-workflow/flow.cylc @@ -1,39 +1,40 @@ -[cylc] - UTC mode = True # Ignore DST - [[parameters]] - county = 1..10 +[task parameters] + county = 1..10 [scheduling] - [[dependencies]] - graph = """ + [[graph]] + # R1: Means run once + # P1: Means run every cycle + R1 = """ clone_repo => get_counties - get_counties => process - process => plot""" + get_counties => process + process => plot + """ [runtime] [[root]] # This is applied before every task pre-script="""set -eu""" - [[[job]]] - batch system = slurm - execution time limit = PT10M + execution time limit = PT10M + platform = lotus # See options via `cylc config --platforms`, e.g. `localhost`, `lotus` [[[directives]]] - --partition = workshop + --partition = debug + --qos = debug --account = workshop [[clone_repo]] script = """rm -fr workshop-repo git clone https://github.com/cedadev/jasmin-workshop workshop-repo -cp workshop-repo/tutorials/tut02/code/* ${CYLC_SUITE_RUN_DIR}/ +cp workshop-repo/tutorials/tut02/code/* ${CYLC_WORKFLOW_RUN_DIR}/ rm -fr workshop-repo""" [[get_counties]] script = """module load jaspy -cd ${CYLC_SUITE_RUN_DIR}/ +cd ${CYLC_WORKFLOW_RUN_DIR}/ python create-counties-file.py""" [[process]] script = """module load jaspy -cd ${CYLC_SUITE_RUN_DIR}/ +cd ${CYLC_WORKFLOW_RUN_DIR}/ python extract-annual-max-series.py ${CYLC_TASK_PARAM_county}""" [[plot]] script = """module load jaspy -cd ${CYLC_SUITE_RUN_DIR}/ +cd ${CYLC_WORKFLOW_RUN_DIR}/ python plot-county-temps.py""" diff --git a/tutorials/tut02/workshop-suite/rose-suite.conf b/tutorials/tut02/workshop-workflow/rose-suite.conf similarity index 100% rename from tutorials/tut02/workshop-suite/rose-suite.conf rename to tutorials/tut02/workshop-workflow/rose-suite.conf diff --git a/tutorials/tut02/workshop-suite/rose-suite.info b/tutorials/tut02/workshop-workflow/rose-suite.info similarity index 100% rename from tutorials/tut02/workshop-suite/rose-suite.info rename to tutorials/tut02/workshop-workflow/rose-suite.info