From a3c4a10f4c5b73c47428c370136e07538bdb688b Mon Sep 17 00:00:00 2001 From: Ruiyang Ding Date: Tue, 28 May 2019 10:17:16 +0300 Subject: [PATCH 1/8] Allow user to set path on host mechine * Using environment variable * Environment variable can give docker backend the location of source file on host machine now. * -p flag is still usable, but will be ignored if environment varible is set. --- apluslms_roman/backends/docker.py | 9 ++++++--- apluslms_roman/builder.py | 6 +++--- apluslms_roman/cli.py | 5 +---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apluslms_roman/backends/docker.py b/apluslms_roman/backends/docker.py index 885ddd6..61c4b68 100644 --- a/apluslms_roman/backends/docker.py +++ b/apluslms_roman/backends/docker.py @@ -29,6 +29,9 @@ def _client(self): timeout = env.get('DOCKER_TIMEOUT', None) if timeout: kwargs['timeout'] = timeout + host_path = env.get('DOCKER_HOST_PATH', None) + if host_path: + print("Host source file path is", host_path) return docker.from_env(environment=env, **kwargs) def _run_opts(self, task, step): @@ -43,14 +46,14 @@ def _run_opts(self, task, step): # mounts and workdir if step.mnt: - opts['mounts'] = [Mount(step.mnt, task.path, type='bind', read_only=False)] + opts['mounts'] = [Mount(step.mnt, env.environ.get('DOCKER_HOST_PATH', task.path), type='bind', read_only=False)] opts['working_dir'] = step.mnt else: wpath = self.WORK_PATH opts['mounts'] = [ Mount(wpath, None, type='tmpfs', read_only=False, tmpfs_size=self.WORK_SIZE), - Mount(join(wpath, 'src'), task.path, type='bind', read_only=True), - Mount(join(wpath, 'build'), join(task.path, '_build'), type='bind', read_only=False), + Mount(join(wpath, 'src'), env.environ.get('DOCKER_HOST_PATH', task.path), type='bind', read_only=True), + Mount(join(wpath, 'build'), join(env.environ.get('DOCKER_HOST_PATH', task.path), '_build'), type='bind', read_only=False), ] opts['working_dir'] = wpath diff --git a/apluslms_roman/builder.py b/apluslms_roman/builder.py index 50e935d..4174f64 100644 --- a/apluslms_roman/builder.py +++ b/apluslms_roman/builder.py @@ -9,6 +9,7 @@ from .utils.importing import import_string from .utils.translation import _ + class Builder: def __init__(self, engine, config, observer=None): if not isdir(config.dir): @@ -18,7 +19,6 @@ def __init__(self, engine, config, observer=None): self._engine = engine self._observer = observer or StreamObserver() - def get_steps(self, refs: list = None): steps = [BuildStep.from_config(i, step) for i, step in enumerate(self.config.steps)] if refs: @@ -32,11 +32,11 @@ def get_steps(self, refs: list = None): raise ValueError(_("A step index was too big. Remember, indexing begins with 0.")) return list(OrderedDict.fromkeys(steps)) - def build(self, step_refs: list = None): + def build(self, step_refs: list = None, host_path=None): backend = self._engine.backend observer = self._observer steps = self.get_steps(step_refs) - task = BuildTask(self.path, steps) + task = BuildTask(self.path if host_path is None else host_path, steps) observer.enter_prepare() backend.prepare(task, observer) observer.enter_build() diff --git a/apluslms_roman/cli.py b/apluslms_roman/cli.py index 90503df..bcc3554 100644 --- a/apluslms_roman/cli.py +++ b/apluslms_roman/cli.py @@ -294,11 +294,8 @@ def main(): exit(context.run()) - ## Actions - # action utils - def get_engine(context): try: return Engine(settings=context.settings) @@ -341,7 +338,6 @@ def build_action(context): config = get_config(context) engine = get_engine(context) builder = engine.create_builder(config) - if context.args.list_steps: steps = builder.get_steps() num_len = max(2, len(str(len(steps)-1))) @@ -436,5 +432,6 @@ def backend_test_action(context, verbose=False): print(engine.version_info()) return 0 + if __name__ == '__main__': main() From e077d7ebb7bb073ff54055d209fded35776263d0 Mon Sep 17 00:00:00 2001 From: Ruiyang Ding Date: Mon, 3 Jun 2019 11:53:08 +0300 Subject: [PATCH 2/8] Support setting path with global config reading from *.yml file --- apluslms_roman/backends/__init__.py | 6 +++++ apluslms_roman/backends/docker.py | 15 +++++------ apluslms_roman/builder.py | 25 +++++++++++++++++-- apluslms_roman/roman_config,yml | 7 ++++++ .../schemas/roman_settings-v1.0.yaml | 4 +++ apluslms_roman/utils/path_mapping.py | 17 +++++++++++++ 6 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 apluslms_roman/roman_config,yml create mode 100644 apluslms_roman/utils/path_mapping.py diff --git a/apluslms_roman/backends/__init__.py b/apluslms_roman/backends/__init__.py index 6d28f15..7f3ef14 100644 --- a/apluslms_roman/backends/__init__.py +++ b/apluslms_roman/backends/__init__.py @@ -107,3 +107,9 @@ def verify(self): def version_info(self): pass + + def remap_path(self, path): + map_ = self.environment.environ.get('directory_map', {}) + logger.debug("get mapping from environment:{}".format(map_)) + map_ = dict(map_) if len(map_) == 0 else map_ + return get_host_path(path, map_) diff --git a/apluslms_roman/backends/docker.py b/apluslms_roman/backends/docker.py index 61c4b68..4ff5e2a 100644 --- a/apluslms_roman/backends/docker.py +++ b/apluslms_roman/backends/docker.py @@ -29,9 +29,9 @@ def _client(self): timeout = env.get('DOCKER_TIMEOUT', None) if timeout: kwargs['timeout'] = timeout - host_path = env.get('DOCKER_HOST_PATH', None) - if host_path: - print("Host source file path is", host_path) + # host_path = env.get('DOCKER_HOST_PATH', None) + # if host_path: + # print("Host source file path is", host_path) return docker.from_env(environment=env, **kwargs) def _run_opts(self, task, step): @@ -43,17 +43,18 @@ def _run_opts(self, task, step): environment=step.env, user='{}:{}'.format(env.uid, env.gid), ) - + path = self.remap_path(task.path) + logger.debug("Final path is:{}".format(path)) # mounts and workdir if step.mnt: - opts['mounts'] = [Mount(step.mnt, env.environ.get('DOCKER_HOST_PATH', task.path), type='bind', read_only=False)] + opts['mounts'] = [Mount(step.mnt, path, type='bind', read_only=False)] opts['working_dir'] = step.mnt else: wpath = self.WORK_PATH opts['mounts'] = [ Mount(wpath, None, type='tmpfs', read_only=False, tmpfs_size=self.WORK_SIZE), - Mount(join(wpath, 'src'), env.environ.get('DOCKER_HOST_PATH', task.path), type='bind', read_only=True), - Mount(join(wpath, 'build'), join(env.environ.get('DOCKER_HOST_PATH', task.path), '_build'), type='bind', read_only=False), + Mount(join(wpath, 'src'), path, type='bind', read_only=True), + Mount(join(wpath, 'build'), join(path, '_build'), type='bind', read_only=False), ] opts['working_dir'] = wpath diff --git a/apluslms_roman/builder.py b/apluslms_roman/builder.py index 4174f64..3f7625f 100644 --- a/apluslms_roman/builder.py +++ b/apluslms_roman/builder.py @@ -4,6 +4,7 @@ from apluslms_yamlidator.utils.decorator import cached_property from apluslms_yamlidator.utils.collections import OrderedDict +from apluslms_roman.utils.path_mapping import get_host_path from .backends import BACKENDS, BuildTask, BuildStep, Environment from .observer import StreamObserver from .utils.importing import import_string @@ -32,11 +33,31 @@ def get_steps(self, refs: list = None): raise ValueError(_("A step index was too big. Remember, indexing begins with 0.")) return list(OrderedDict.fromkeys(steps)) - def build(self, step_refs: list = None, host_path=None): + def build(self, step_refs: list = None): backend = self._engine.backend observer = self._observer steps = self.get_steps(step_refs) - task = BuildTask(self.path if host_path is None else host_path, steps) + # Check if has global config set. + # Using path in global config file + print("dir mapping set in config:", self._engine._dir_mapping if hasattr(self._engine, '_dir_mapping') else None) + print("env set:", self._engine._environment.environ.get('DOCKER_HOST_PATH', None)) + + if hasattr(self._engine, '_dir_mapping'): + print("Using roman config") + path = get_host_path(self.path, self._engine._dir_mapping) + elif self._engine._environment.environ.get('DOCKER_HOST_PATH', None) is not None: + print("Using DOCKER_HOST_PATH") + path = self._engine._environment.environ.get('DOCKER_HOST_PATH', None) + else: + path = self.path + print("No config find") + task = BuildTask(path, steps) + # Using config file + # Using -p flag or global environment variable + # task = BuildTask(self.path if host_path is None else host_path, steps) + + task = BuildTask(self.path, steps) + observer.enter_prepare() backend.prepare(task, observer) observer.enter_build() diff --git a/apluslms_roman/roman_config,yml b/apluslms_roman/roman_config,yml new file mode 100644 index 0000000..a1fb940 --- /dev/null +++ b/apluslms_roman/roman_config,yml @@ -0,0 +1,7 @@ + +version: '1.0' +backend: docker + +docker: + directory_map: + /data/: /var/lib/docker/volumes/aplus_data/_data/ \ No newline at end of file diff --git a/apluslms_roman/schemas/roman_settings-v1.0.yaml b/apluslms_roman/schemas/roman_settings-v1.0.yaml index a972077..7a5865b 100644 --- a/apluslms_roman/schemas/roman_settings-v1.0.yaml +++ b/apluslms_roman/schemas/roman_settings-v1.0.yaml @@ -24,6 +24,10 @@ properties: type: object additionalProperties: false properties: + directory_map: + title: docker conatiner-host machine path mapping + description: The dictornary mapping between docker and it's host + type: object host: title: docker host description: the URL to the Docker host diff --git a/apluslms_roman/utils/path_mapping.py b/apluslms_roman/utils/path_mapping.py new file mode 100644 index 0000000..83d8874 --- /dev/null +++ b/apluslms_roman/utils/path_mapping.py @@ -0,0 +1,17 @@ +from pathlib import PurePosixPath + + +def get_host_path(original, mapping): + ret = original + orig_path = PurePosixPath(original) + for k, v in mapping.items(): + print(k, v) + try: + relative_path = orig_path.relative_to(k) + ret = PurePosixPath(v).joinpath(relative_path) + print("Get new path:", ret) + return str(ret) + except ValueError: + pass + return str(ret) + From 9fe5e454788b482f32041a102ebb9b1492c62b22 Mon Sep 17 00:00:00 2001 From: Ruiyang Ding Date: Thu, 13 Jun 2019 12:48:10 +0300 Subject: [PATCH 3/8] Add utiles which read settings from environemt variable --- apluslms_roman/builder.py | 20 ++++----- apluslms_roman/utils/path_mapping.py | 62 +++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/apluslms_roman/builder.py b/apluslms_roman/builder.py index 3f7625f..ab400fc 100644 --- a/apluslms_roman/builder.py +++ b/apluslms_roman/builder.py @@ -2,13 +2,12 @@ from os.path import isdir, join from apluslms_yamlidator.utils.decorator import cached_property -from apluslms_yamlidator.utils.collections import OrderedDict +from apluslms_yamlidator.utils.collections import OrderedDict, ChangesDict +from apluslms_roman.utils.path_mapping import load_from_env, get_pair_form_env -from apluslms_roman.utils.path_mapping import get_host_path from .backends import BACKENDS, BuildTask, BuildStep, Environment from .observer import StreamObserver from .utils.importing import import_string -from .utils.translation import _ class Builder: @@ -38,7 +37,6 @@ def build(self, step_refs: list = None): observer = self._observer steps = self.get_steps(step_refs) # Check if has global config set. - # Using path in global config file print("dir mapping set in config:", self._engine._dir_mapping if hasattr(self._engine, '_dir_mapping') else None) print("env set:", self._engine._environment.environ.get('DOCKER_HOST_PATH', None)) @@ -48,15 +46,13 @@ def build(self, step_refs: list = None): elif self._engine._environment.environ.get('DOCKER_HOST_PATH', None) is not None: print("Using DOCKER_HOST_PATH") path = self._engine._environment.environ.get('DOCKER_HOST_PATH', None) + + else: path = self.path print("No config find") + print("The final composed host path is:", path) task = BuildTask(path, steps) - # Using config file - # Using -p flag or global environment variable - # task = BuildTask(self.path if host_path is None else host_path, steps) - - task = BuildTask(self.path, steps) observer.enter_prepare() backend.prepare(task, observer) @@ -81,11 +77,13 @@ def __init__(self, backend_class=None, settings=None): name = getattr(backend_class, 'name', None) or backend_class.__name__.lower() env_prefix = name.upper() + '_' - env = {k: v for k, v in environ.items() if k.startswith(env_prefix)} + env = load_from_env(env_prefix, '.') + print("env without setting:", env) if settings: for k, v in settings.get(name, {}).items(): if v is not None and v != '': - env[env_prefix + k.replace('-', '_').upper()] = v + env[k] = dict(v) if isinstance(v, ChangesDict) else v + print("env after setting:", env) self._environment = Environment(getuid(), getegid(), env) @cached_property diff --git a/apluslms_roman/utils/path_mapping.py b/apluslms_roman/utils/path_mapping.py index 83d8874..bd4ac06 100644 --- a/apluslms_roman/utils/path_mapping.py +++ b/apluslms_roman/utils/path_mapping.py @@ -1,17 +1,75 @@ from pathlib import PurePosixPath +import json +import logging +import re +from os import environ + +logger = logging.getLogger(__name__) + def get_host_path(original, mapping): ret = original orig_path = PurePosixPath(original) for k, v in mapping.items(): - print(k, v) try: + logger.debug("Mapping:{}:{}".format(k, v)) relative_path = orig_path.relative_to(k) ret = PurePosixPath(v).joinpath(relative_path) - print("Get new path:", ret) return str(ret) except ValueError: + logger.critical("Error when composing new path!") pass return str(ret) + +def get_pair_form_env(key, json_str, read_key=None): + if key == read_key: + try: + ret = json.loads(json_str) + if isinstance(ret, dict): + return ret + except json.decoder.JSONDecodeError: + logger.debug("Error, check your json string") + return json_str + + +def env_process_key(key, prefix): + return + + +def env_value_to_dict(json_str): + if re.match(r'^(?:["[{]|(?:\d+|null|true|false)$)', json_str): + try: + ret = json.loads(json_str) + if isinstance(ret, dict): + return ret + except json.decoder.JSONDecodeError: + logger.debug("Error, check your json string") + + return json_str + + +def nest_dict(flat_dict, sep): + ret = {} + for k, v in flat_dict.items(): + key_list = k.split(sep, 1) + if len(key_list) == 2: + root = key_list[0] + if root not in ret: + ret[root] = {} + ret[root][key_list[1]] = v + else: + ret[k] = v + return ret + + +def load_from_env(env_prefix=None, sep=None, decode_json=True): + if decode_json: + decode = lambda s: json.loads(s) if json_re.match(s) is not None else s + else: + decode = lambda s: s + env = {k[len(env_prefix):].lower(): decode(v) for k, v in environ.items() if k.startswith(env_prefix)} + if sep is not None: + env = nest_dict(env, sep) + return env \ No newline at end of file From c89209ac8687492ad73bda467900d7b1ed59e65b Mon Sep 17 00:00:00 2001 From: Ruiyang Ding Date: Thu, 13 Jun 2019 12:54:04 +0300 Subject: [PATCH 4/8] Using different way to init DockerClient --- apluslms_roman/backends/docker.py | 34 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/apluslms_roman/backends/docker.py b/apluslms_roman/backends/docker.py index 4ff5e2a..e013cd0 100644 --- a/apluslms_roman/backends/docker.py +++ b/apluslms_roman/backends/docker.py @@ -2,6 +2,7 @@ from os.path import join from apluslms_yamlidator.utils.decorator import cached_property +from docker import DockerClient from ..utils.translation import _ from . import ( @@ -22,17 +23,28 @@ class DockerBackend(Backend): @cached_property def _client(self): env = self.environment.environ - kwargs = {} - version = env.get('DOCKER_VERSION', None) - if version: - kwargs['version'] = version - timeout = env.get('DOCKER_TIMEOUT', None) - if timeout: - kwargs['timeout'] = timeout - # host_path = env.get('DOCKER_HOST_PATH', None) - # if host_path: - # print("Host source file path is", host_path) - return docker.from_env(environment=env, **kwargs) + params = { + 'base_url': env.get('host'), + 'version': env.get('version'), + } + if 'timeout' in env: + params['timeout'] = env['timeout'] + + # false values: 0, false, '', None + # true values: 1, true, "yes", unset + tls_verify = bool(env.get('tls_verify', False)) + cert_path = env.get('cert_path') or None + if tls_verify or cert_path: + if not cert_path: + cert_path = os.path.join(os.path.expanduser('~'), '.docker') + params['tls'] = docker.tls.TLSConfig( + client_cert=(os.path.join(cert_path, 'cert.pem'), os.path.join(cert_path, 'key.pem')), + ca_cert=os.path.join(cert_path, 'ca.pem'), + verify=tls_verify, + ssl_version=env.get('tls_ssl_version'), + assert_hostname=tls_verify and env.get('tls_assert_hostname'), + ) + return DockerClient(**params) def _run_opts(self, task, step): env = self.environment From 83e17199c7908e47e1b28db72d1e4d9022c1ed4f Mon Sep 17 00:00:00 2001 From: Ruiyang Ding Date: Thu, 13 Jun 2019 12:55:14 +0300 Subject: [PATCH 5/8] Remove global cofig file from repo --- apluslms_roman/builder.py | 40 ++++++++++++--------------------- apluslms_roman/roman_config,yml | 7 ------ setup.py | 0 3 files changed, 14 insertions(+), 33 deletions(-) delete mode 100644 apluslms_roman/roman_config,yml mode change 100755 => 100644 setup.py diff --git a/apluslms_roman/builder.py b/apluslms_roman/builder.py index ab400fc..f6eb0c9 100644 --- a/apluslms_roman/builder.py +++ b/apluslms_roman/builder.py @@ -1,13 +1,18 @@ -from os import environ, getuid, getegid -from os.path import isdir, join +import logging +from os import getuid, getegid +from os.path import isdir +from apluslms_yamlidator.utils.collections import OrderedDict from apluslms_yamlidator.utils.decorator import cached_property -from apluslms_yamlidator.utils.collections import OrderedDict, ChangesDict -from apluslms_roman.utils.path_mapping import load_from_env, get_pair_form_env +from apluslms_roman.utils.path_mapping import load_from_env from .backends import BACKENDS, BuildTask, BuildStep, Environment from .observer import StreamObserver from .utils.importing import import_string +from .utils.translation import _ + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) class Builder: @@ -36,24 +41,7 @@ def build(self, step_refs: list = None): backend = self._engine.backend observer = self._observer steps = self.get_steps(step_refs) - # Check if has global config set. - print("dir mapping set in config:", self._engine._dir_mapping if hasattr(self._engine, '_dir_mapping') else None) - print("env set:", self._engine._environment.environ.get('DOCKER_HOST_PATH', None)) - - if hasattr(self._engine, '_dir_mapping'): - print("Using roman config") - path = get_host_path(self.path, self._engine._dir_mapping) - elif self._engine._environment.environ.get('DOCKER_HOST_PATH', None) is not None: - print("Using DOCKER_HOST_PATH") - path = self._engine._environment.environ.get('DOCKER_HOST_PATH', None) - - - else: - path = self.path - print("No config find") - print("The final composed host path is:", path) - task = BuildTask(path, steps) - + task = BuildTask(self.path, steps) observer.enter_prepare() backend.prepare(task, observer) observer.enter_build() @@ -78,12 +66,12 @@ def __init__(self, backend_class=None, settings=None): name = getattr(backend_class, 'name', None) or backend_class.__name__.lower() env_prefix = name.upper() + '_' env = load_from_env(env_prefix, '.') - print("env without setting:", env) + logger.debug("env without reading global config:{}".format(env)) if settings: for k, v in settings.get(name, {}).items(): if v is not None and v != '': - env[k] = dict(v) if isinstance(v, ChangesDict) else v - print("env after setting:", env) + env[k] = v + logger.debug("env after read from global config:{}".format(env)) self._environment = Environment(getuid(), getegid(), env) @cached_property @@ -97,4 +85,4 @@ def version_info(self): return self.backend.version_info() def create_builder(self, *args, **kwargs): - return Builder(self, *args, **kwargs) + return Builder(self, *args, **kwargs) \ No newline at end of file diff --git a/apluslms_roman/roman_config,yml b/apluslms_roman/roman_config,yml deleted file mode 100644 index a1fb940..0000000 --- a/apluslms_roman/roman_config,yml +++ /dev/null @@ -1,7 +0,0 @@ - -version: '1.0' -backend: docker - -docker: - directory_map: - /data/: /var/lib/docker/volumes/aplus_data/_data/ \ No newline at end of file diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 From 13d8a605ff61851c7e3d303d6c7ce57a9f223033 Mon Sep 17 00:00:00 2001 From: Ruiyang Ding Date: Thu, 13 Jun 2019 13:51:07 +0300 Subject: [PATCH 6/8] Using logging instead of print --- apluslms_roman/backends/__init__.py | 5 +++++ apluslms_roman/backends/docker.py | 5 ++++- apluslms_roman/utils/path_mapping.py | 18 +++++++----------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/apluslms_roman/backends/__init__.py b/apluslms_roman/backends/__init__.py index 7f3ef14..d3ca279 100644 --- a/apluslms_roman/backends/__init__.py +++ b/apluslms_roman/backends/__init__.py @@ -1,6 +1,8 @@ +import logging from collections import namedtuple from collections.abc import Mapping +from apluslms_roman.utils.path_mapping import get_host_path from ..observer import BuildObserver @@ -15,6 +17,9 @@ ]) +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + def clean_image_name(image): if ':' not in image: image += ':latest' diff --git a/apluslms_roman/backends/docker.py b/apluslms_roman/backends/docker.py index e013cd0..61d42fe 100644 --- a/apluslms_roman/backends/docker.py +++ b/apluslms_roman/backends/docker.py @@ -1,3 +1,5 @@ +import os +import logging import docker from os.path import join @@ -10,7 +12,8 @@ BuildResult, ) - +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) Mount = docker.types.Mount diff --git a/apluslms_roman/utils/path_mapping.py b/apluslms_roman/utils/path_mapping.py index bd4ac06..3d447c4 100644 --- a/apluslms_roman/utils/path_mapping.py +++ b/apluslms_roman/utils/path_mapping.py @@ -4,8 +4,10 @@ import re from os import environ -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +json_re = re.compile(r'^(?:["[{]|(?:-?[1-9]\d*(?:\.\d+)?|null|true|false)$)') def get_host_path(original, mapping): @@ -30,23 +32,17 @@ def get_pair_form_env(key, json_str, read_key=None): if isinstance(ret, dict): return ret except json.decoder.JSONDecodeError: - logger.debug("Error, check your json string") + logger.critical("Error, check your json string") return json_str -def env_process_key(key, prefix): - return - - def env_value_to_dict(json_str): - if re.match(r'^(?:["[{]|(?:\d+|null|true|false)$)', json_str): + if json_re.match(json_str): try: ret = json.loads(json_str) - if isinstance(ret, dict): - return ret + return ret except json.decoder.JSONDecodeError: - logger.debug("Error, check your json string") - + logger.critical("Error, check your json string") return json_str From 079bb6019032d8845b728a44c5fb5302fde8a04d Mon Sep 17 00:00:00 2001 From: Ruiyang Ding Date: Thu, 13 Jun 2019 14:49:39 +0300 Subject: [PATCH 7/8] Add unittests --- tests/utils/test_path_mapping.py | 63 ++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/utils/test_path_mapping.py diff --git a/tests/utils/test_path_mapping.py b/tests/utils/test_path_mapping.py new file mode 100644 index 0000000..3ebecb7 --- /dev/null +++ b/tests/utils/test_path_mapping.py @@ -0,0 +1,63 @@ +import unittest +from json import loads, dumps +from apluslms_roman.utils.path_mapping import json_re, env_value_to_dict + +test_case_loadable = ( + 'true', + 'false', + 'null', + '123', + '-123', + '3.14', + '-3.14', + '{"foo": "bar"}', + '[1, 2, 3]', + '"foo bar"' +) + +test_case_not_loadable = ( + "/foobar.py", + "text", + "yes", + "0123123", +) + + +class TestJsonLoadable(unittest.TestCase): + + def test_loadable_not_raise(self): + for i, case in enumerate(test_case_loadable): + with self.subTest(i=i): + loads(case) + + def test_not_loadable_raise(self): + for i, case in enumerate(test_case_not_loadable): + with self.subTest(i=i): + with self.assertRaises(ValueError, msg="Testing:{}".format(case)): + loads(case) + + +class TestJsonRegex(unittest.TestCase): + + def test_loadable_match(self): + for i, case in enumerate(test_case_loadable): + with self.subTest(i=i): + self.assertTrue(json_re.match(case)is not None, msg="Testing:{}".format(case)) + + def test_not_loadable_not_match(self): + for i, case in enumerate(test_case_not_loadable): + with self.subTest(i=i): + self.assertFalse(json_re.match(case) is not None, msg="Testing:{}".format(case)) + + +class TestValueDict(unittest.TestCase): + + def test_loadable_type(self): + for i, case in enumerate(test_case_loadable): + with self.subTest(i=i): + self.assertEqual(env_value_to_dict(case), loads(case), msg="Testing:{}".format(case)) + + def test_not_loadable_type(self): + for i, case in enumerate(test_case_not_loadable): + with self.subTest(i=i): + self.assertEqual(env_value_to_dict(case), case, msg="Testing:{}".format(case)) From c4d695726a2d64bc28d2b757dd02eeac48169793 Mon Sep 17 00:00:00 2001 From: Ruiyang Ding Date: Thu, 13 Jun 2019 16:57:02 +0300 Subject: [PATCH 8/8] Add Kubernetes backend for course material build. --- apluslms_roman/backends/__init__.py | 1 + apluslms_roman/backends/kubernetes.py | 135 ++++++++++++++++++ .../schemas/roman_settings-v1.0.yaml | 11 ++ apluslms_roman/utils/kubernetes.py | 33 +++++ requirements.txt | 1 + tests/utils/test_kubernetes.py | 39 +++++ 6 files changed, 220 insertions(+) create mode 100644 apluslms_roman/backends/kubernetes.py create mode 100644 apluslms_roman/utils/kubernetes.py create mode 100644 tests/utils/test_kubernetes.py diff --git a/apluslms_roman/backends/__init__.py b/apluslms_roman/backends/__init__.py index d3ca279..77d430f 100644 --- a/apluslms_roman/backends/__init__.py +++ b/apluslms_roman/backends/__init__.py @@ -8,6 +8,7 @@ BACKENDS = { 'docker': 'apluslms_roman.backends.docker.DockerBackend', + 'kubernetes': 'apluslms_roman.backends.kubernetes.KubernetesBackend', } diff --git a/apluslms_roman/backends/kubernetes.py b/apluslms_roman/backends/kubernetes.py new file mode 100644 index 0000000..04b9b97 --- /dev/null +++ b/apluslms_roman/backends/kubernetes.py @@ -0,0 +1,135 @@ +import logging + +from os.path import join +from apluslms_yamlidator.utils.decorator import cached_property +from kubernetes import client, config, watch +from apluslms_roman.utils.kubernetes import create_pod +from apluslms_roman.backends import BuildTask +from apluslms_roman.observer import BuildObserver +from . import ( + Backend, + BuildResult, +) + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +class KubernetesBackend(Backend): + """ + Run each step as a Kubernetes Deployment + Mounting: using mounting in deployment, mapping is same as shepherd + """ + name = 'kubernetes' + + @cached_property + def _client(self): + # Load kubernetes config from from $Home/.kube/config + config.load_kube_config() + api = client.CoreV1Api() + return api + + def _run_opts(self, task, step): + """ + Define the Pod model + """ + env = self.environment + opts = dict( + image=step.img, + command=step.cmd, + environment=step.env, + namespace=env.environ['namespace'], + name=step.img.split(':')[0].replace('/', '-') + ) + if step.mnt: + opts['volumes'] = [ + client.V1Volume( + name='build-path', + host_path=client.V1HostPathVolumeSource(path="/build-source") + ) + ] + opts['mounts'] = [ + client.V1VolumeMount( + mount_path=step.mnt, + name='build-path' + ) + ] + opts['working_dir'] = step.mnt + else: + wpath = self.WORK_PATH + + opts['volumes'] = [ + client.V1Volume( + name='cache', + empty_dir=client.V1EmptyDirVolumeSource(size_limit=self.WORK_SIZE, medium='Memory') + ), + client.V1Volume( + name='source', + host_path=client.V1HostPathVolumeSource(path=join(wpath, 'src')) + ), + client.V1Volume( + name='build', + host_path=client.V1HostPathVolumeSource(path=join(wpath, 'build')) + ) + ] + opts['mounts'] = [ + client.V1VolumeMount( + mount_path=wpath, + name='cache', + read_only=False + ), + client.V1VolumeMount( + mount_path=join(wpath, 'src'), + name='source', + read_only=True + ), + client.V1VolumeMount( + mount_path=join(wpath, 'build'), + name='build', + read_only=False + ) + ] + return opts + + def prepare(self, task: BuildTask, observer: BuildObserver): + pass + + def build(self, task: BuildTask, observer: BuildObserver): + api_client = self._client + for step in task.steps: + observer.start_step(step) + opts = self._run_opts(task, step) + observer.manager_msg(step, "Running deployment with image {}:".format(opts['image'])) + name = opts['name'] + try: + create_resp = create_pod(**opts) + print(create_resp) + name = create_resp.metadata.name + # Waiting pod finished + while True: + resp = api_client.read_namespaced_pod(name=name, namespace=opts['namespace']) + if resp.status.phase != "Pending": + break + for line in api_client.read_namespaced_pod_log( + name=name, + namespace=opts['namespace'], + follow=True, + _preload_content=False).stream(): + observer.container_msg(step, line.decode('utf-8')) + except client.rest.ApiException as e: + logger.warning('Error when create Pod: %s.\n' % e) + return BuildResult(1, e, step) + finally: + api_client.delete_namespaced_pod( + name=name, + namespace=opts['namespace'], + ) + observer.end_step(step) + return BuildResult() + + def verify(self): + try: + api_client = self._client + api_client.list_component_status() + except Exception as e: + return "{}: {}".format(e.__class__.__name__, e) diff --git a/apluslms_roman/schemas/roman_settings-v1.0.yaml b/apluslms_roman/schemas/roman_settings-v1.0.yaml index 7a5865b..a774d41 100644 --- a/apluslms_roman/schemas/roman_settings-v1.0.yaml +++ b/apluslms_roman/schemas/roman_settings-v1.0.yaml @@ -9,6 +9,7 @@ allOf: optional: - backend - docker + - kubernetes additionalProperties: false properties: @@ -50,3 +51,13 @@ properties: type: integer minimum: 0 exclusiveMinimum: true + kubernetes: + title: kubernetes backend options + description: options for Kubernetes backend + type: object + additionalProperties: false + properties: + namespace: + title: default pod namespace + description: default namesapce for generated pod which runs build container. + type: string \ No newline at end of file diff --git a/apluslms_roman/utils/kubernetes.py b/apluslms_roman/utils/kubernetes.py new file mode 100644 index 0000000..d6119ec --- /dev/null +++ b/apluslms_roman/utils/kubernetes.py @@ -0,0 +1,33 @@ +from kubernetes import client +import logging + +logger = logging.getLogger(__name__) + + +def create_pod(image, command, environment, name, namespace, mounts, volumes, working_dir): + container = client.V1Container( + name=name, + image=image, + volume_mounts=mounts, + args=command.split(), + working_dir=working_dir, + env=[client.V1EnvVar(k, v) for k, v in environment.items()], + security_context=client.V1SecurityContext( + privileged=True, + ), + ) + pod = client.V1Pod( + metadata=client.V1ObjectMeta( + generate_name=name+'-', + namespace=namespace, + labels={"app": "roman"}, + ), + spec=client.V1PodSpec( + containers=[container], + volumes=volumes, + restart_policy="Never", + ) + ) + v1 = client.CoreV1Api() + res = v1.create_namespaced_pod(namespace, pod) + return res diff --git a/requirements.txt b/requirements.txt index 4b1ef59..188273f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ apluslms-yamlidator appdirs >=1.4.0, <2 +kubernetes diff --git a/tests/utils/test_kubernetes.py b/tests/utils/test_kubernetes.py new file mode 100644 index 0000000..5c44827 --- /dev/null +++ b/tests/utils/test_kubernetes.py @@ -0,0 +1,39 @@ +import unittest +from apluslms_roman.utils.kubernetes import create_pod +from kubernetes import client, config + +test_cases = [ + { + 'image': 'foo/bar:latest', + 'command': 'ls', + 'environment': {'foo': 'bar'}, + 'name': 'bar', + 'namespace': 'default', + 'mounts': [ + client.V1VolumeMount( + mount_path='/', + name='build-path' + ) + ], + 'volumes': [ + client.V1Volume( + name='build-path', + host_path=client.V1HostPathVolumeSource(path="/build-source") + ) + ], + 'working_dir': '/' + } +] + + +class TestPod(unittest.TestCase): + def test_create_pod(self): + for i, test_case in enumerate(test_cases): + with self.subTest(i=i): + config.load_kube_config() + pod = create_pod(**test_case) + self.assertTrue(pod.metadata.name.startswith(test_case['name'])) + self.assertEqual(pod.metadata.namespace, test_case['namespace']) + self.assertEqual(pod.spec.containers[0].env, [client.V1EnvVar(k, v) for k, v in {'foo': 'bar'}.items()]) + self.assertEqual(pod.spec.containers[0].working_dir, test_case['working_dir']) + self.assertEqual(pod.spec.containers[0].volume_mounts[0].name, test_case['volumes'][0].name)