diff --git a/apluslms_roman/backends/__init__.py b/apluslms_roman/backends/__init__.py index 6d28f15..77d430f 100644 --- a/apluslms_roman/backends/__init__.py +++ b/apluslms_roman/backends/__init__.py @@ -1,11 +1,14 @@ +import logging from collections import namedtuple from collections.abc import Mapping +from apluslms_roman.utils.path_mapping import get_host_path from ..observer import BuildObserver BACKENDS = { 'docker': 'apluslms_roman.backends.docker.DockerBackend', + 'kubernetes': 'apluslms_roman.backends.kubernetes.KubernetesBackend', } @@ -15,6 +18,9 @@ ]) +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + def clean_image_name(image): if ':' not in image: image += ':latest' @@ -107,3 +113,9 @@ def verify(self): def version_info(self): pass + + def remap_path(self, path): + map_ = self.environment.environ.get('directory_map', {}) + logger.debug("get mapping from environment:{}".format(map_)) + map_ = dict(map_) if len(map_) == 0 else map_ + return get_host_path(path, map_) diff --git a/apluslms_roman/backends/docker.py b/apluslms_roman/backends/docker.py index 885ddd6..61d42fe 100644 --- a/apluslms_roman/backends/docker.py +++ b/apluslms_roman/backends/docker.py @@ -1,7 +1,10 @@ +import os +import logging import docker from os.path import join from apluslms_yamlidator.utils.decorator import cached_property +from docker import DockerClient from ..utils.translation import _ from . import ( @@ -9,7 +12,8 @@ BuildResult, ) - +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) Mount = docker.types.Mount @@ -22,14 +26,28 @@ class DockerBackend(Backend): @cached_property def _client(self): env = self.environment.environ - kwargs = {} - version = env.get('DOCKER_VERSION', None) - if version: - kwargs['version'] = version - timeout = env.get('DOCKER_TIMEOUT', None) - if timeout: - kwargs['timeout'] = timeout - return docker.from_env(environment=env, **kwargs) + params = { + 'base_url': env.get('host'), + 'version': env.get('version'), + } + if 'timeout' in env: + params['timeout'] = env['timeout'] + + # false values: 0, false, '', None + # true values: 1, true, "yes", unset + tls_verify = bool(env.get('tls_verify', False)) + cert_path = env.get('cert_path') or None + if tls_verify or cert_path: + if not cert_path: + cert_path = os.path.join(os.path.expanduser('~'), '.docker') + params['tls'] = docker.tls.TLSConfig( + client_cert=(os.path.join(cert_path, 'cert.pem'), os.path.join(cert_path, 'key.pem')), + ca_cert=os.path.join(cert_path, 'ca.pem'), + verify=tls_verify, + ssl_version=env.get('tls_ssl_version'), + assert_hostname=tls_verify and env.get('tls_assert_hostname'), + ) + return DockerClient(**params) def _run_opts(self, task, step): env = self.environment @@ -40,17 +58,18 @@ def _run_opts(self, task, step): environment=step.env, user='{}:{}'.format(env.uid, env.gid), ) - + path = self.remap_path(task.path) + logger.debug("Final path is:{}".format(path)) # mounts and workdir if step.mnt: - opts['mounts'] = [Mount(step.mnt, task.path, type='bind', read_only=False)] + opts['mounts'] = [Mount(step.mnt, path, type='bind', read_only=False)] opts['working_dir'] = step.mnt else: wpath = self.WORK_PATH opts['mounts'] = [ Mount(wpath, None, type='tmpfs', read_only=False, tmpfs_size=self.WORK_SIZE), - Mount(join(wpath, 'src'), task.path, type='bind', read_only=True), - Mount(join(wpath, 'build'), join(task.path, '_build'), type='bind', read_only=False), + Mount(join(wpath, 'src'), path, type='bind', read_only=True), + Mount(join(wpath, 'build'), join(path, '_build'), type='bind', read_only=False), ] opts['working_dir'] = wpath diff --git a/apluslms_roman/backends/kubernetes.py b/apluslms_roman/backends/kubernetes.py new file mode 100644 index 0000000..04b9b97 --- /dev/null +++ b/apluslms_roman/backends/kubernetes.py @@ -0,0 +1,135 @@ +import logging + +from os.path import join +from apluslms_yamlidator.utils.decorator import cached_property +from kubernetes import client, config, watch +from apluslms_roman.utils.kubernetes import create_pod +from apluslms_roman.backends import BuildTask +from apluslms_roman.observer import BuildObserver +from . import ( + Backend, + BuildResult, +) + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +class KubernetesBackend(Backend): + """ + Run each step as a Kubernetes Deployment + Mounting: using mounting in deployment, mapping is same as shepherd + """ + name = 'kubernetes' + + @cached_property + def _client(self): + # Load kubernetes config from from $Home/.kube/config + config.load_kube_config() + api = client.CoreV1Api() + return api + + def _run_opts(self, task, step): + """ + Define the Pod model + """ + env = self.environment + opts = dict( + image=step.img, + command=step.cmd, + environment=step.env, + namespace=env.environ['namespace'], + name=step.img.split(':')[0].replace('/', '-') + ) + if step.mnt: + opts['volumes'] = [ + client.V1Volume( + name='build-path', + host_path=client.V1HostPathVolumeSource(path="/build-source") + ) + ] + opts['mounts'] = [ + client.V1VolumeMount( + mount_path=step.mnt, + name='build-path' + ) + ] + opts['working_dir'] = step.mnt + else: + wpath = self.WORK_PATH + + opts['volumes'] = [ + client.V1Volume( + name='cache', + empty_dir=client.V1EmptyDirVolumeSource(size_limit=self.WORK_SIZE, medium='Memory') + ), + client.V1Volume( + name='source', + host_path=client.V1HostPathVolumeSource(path=join(wpath, 'src')) + ), + client.V1Volume( + name='build', + host_path=client.V1HostPathVolumeSource(path=join(wpath, 'build')) + ) + ] + opts['mounts'] = [ + client.V1VolumeMount( + mount_path=wpath, + name='cache', + read_only=False + ), + client.V1VolumeMount( + mount_path=join(wpath, 'src'), + name='source', + read_only=True + ), + client.V1VolumeMount( + mount_path=join(wpath, 'build'), + name='build', + read_only=False + ) + ] + return opts + + def prepare(self, task: BuildTask, observer: BuildObserver): + pass + + def build(self, task: BuildTask, observer: BuildObserver): + api_client = self._client + for step in task.steps: + observer.start_step(step) + opts = self._run_opts(task, step) + observer.manager_msg(step, "Running deployment with image {}:".format(opts['image'])) + name = opts['name'] + try: + create_resp = create_pod(**opts) + print(create_resp) + name = create_resp.metadata.name + # Waiting pod finished + while True: + resp = api_client.read_namespaced_pod(name=name, namespace=opts['namespace']) + if resp.status.phase != "Pending": + break + for line in api_client.read_namespaced_pod_log( + name=name, + namespace=opts['namespace'], + follow=True, + _preload_content=False).stream(): + observer.container_msg(step, line.decode('utf-8')) + except client.rest.ApiException as e: + logger.warning('Error when create Pod: %s.\n' % e) + return BuildResult(1, e, step) + finally: + api_client.delete_namespaced_pod( + name=name, + namespace=opts['namespace'], + ) + observer.end_step(step) + return BuildResult() + + def verify(self): + try: + api_client = self._client + api_client.list_component_status() + except Exception as e: + return "{}: {}".format(e.__class__.__name__, e) diff --git a/apluslms_roman/builder.py b/apluslms_roman/builder.py index 50e935d..f6eb0c9 100644 --- a/apluslms_roman/builder.py +++ b/apluslms_roman/builder.py @@ -1,14 +1,20 @@ -from os import environ, getuid, getegid -from os.path import isdir, join +import logging +from os import getuid, getegid +from os.path import isdir -from apluslms_yamlidator.utils.decorator import cached_property from apluslms_yamlidator.utils.collections import OrderedDict +from apluslms_yamlidator.utils.decorator import cached_property +from apluslms_roman.utils.path_mapping import load_from_env from .backends import BACKENDS, BuildTask, BuildStep, Environment from .observer import StreamObserver from .utils.importing import import_string from .utils.translation import _ +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + class Builder: def __init__(self, engine, config, observer=None): if not isdir(config.dir): @@ -18,7 +24,6 @@ def __init__(self, engine, config, observer=None): self._engine = engine self._observer = observer or StreamObserver() - def get_steps(self, refs: list = None): steps = [BuildStep.from_config(i, step) for i, step in enumerate(self.config.steps)] if refs: @@ -60,11 +65,13 @@ def __init__(self, backend_class=None, settings=None): name = getattr(backend_class, 'name', None) or backend_class.__name__.lower() env_prefix = name.upper() + '_' - env = {k: v for k, v in environ.items() if k.startswith(env_prefix)} + env = load_from_env(env_prefix, '.') + logger.debug("env without reading global config:{}".format(env)) if settings: for k, v in settings.get(name, {}).items(): if v is not None and v != '': - env[env_prefix + k.replace('-', '_').upper()] = v + env[k] = v + logger.debug("env after read from global config:{}".format(env)) self._environment = Environment(getuid(), getegid(), env) @cached_property @@ -78,4 +85,4 @@ def version_info(self): return self.backend.version_info() def create_builder(self, *args, **kwargs): - return Builder(self, *args, **kwargs) + return Builder(self, *args, **kwargs) \ No newline at end of file diff --git a/apluslms_roman/cli.py b/apluslms_roman/cli.py index 90503df..bcc3554 100644 --- a/apluslms_roman/cli.py +++ b/apluslms_roman/cli.py @@ -294,11 +294,8 @@ def main(): exit(context.run()) - ## Actions - # action utils - def get_engine(context): try: return Engine(settings=context.settings) @@ -341,7 +338,6 @@ def build_action(context): config = get_config(context) engine = get_engine(context) builder = engine.create_builder(config) - if context.args.list_steps: steps = builder.get_steps() num_len = max(2, len(str(len(steps)-1))) @@ -436,5 +432,6 @@ def backend_test_action(context, verbose=False): print(engine.version_info()) return 0 + if __name__ == '__main__': main() diff --git a/apluslms_roman/schemas/roman_settings-v1.0.yaml b/apluslms_roman/schemas/roman_settings-v1.0.yaml index a972077..a774d41 100644 --- a/apluslms_roman/schemas/roman_settings-v1.0.yaml +++ b/apluslms_roman/schemas/roman_settings-v1.0.yaml @@ -9,6 +9,7 @@ allOf: optional: - backend - docker + - kubernetes additionalProperties: false properties: @@ -24,6 +25,10 @@ properties: type: object additionalProperties: false properties: + directory_map: + title: docker conatiner-host machine path mapping + description: The dictornary mapping between docker and it's host + type: object host: title: docker host description: the URL to the Docker host @@ -46,3 +51,13 @@ properties: type: integer minimum: 0 exclusiveMinimum: true + kubernetes: + title: kubernetes backend options + description: options for Kubernetes backend + type: object + additionalProperties: false + properties: + namespace: + title: default pod namespace + description: default namesapce for generated pod which runs build container. + type: string \ No newline at end of file diff --git a/apluslms_roman/utils/kubernetes.py b/apluslms_roman/utils/kubernetes.py new file mode 100644 index 0000000..d6119ec --- /dev/null +++ b/apluslms_roman/utils/kubernetes.py @@ -0,0 +1,33 @@ +from kubernetes import client +import logging + +logger = logging.getLogger(__name__) + + +def create_pod(image, command, environment, name, namespace, mounts, volumes, working_dir): + container = client.V1Container( + name=name, + image=image, + volume_mounts=mounts, + args=command.split(), + working_dir=working_dir, + env=[client.V1EnvVar(k, v) for k, v in environment.items()], + security_context=client.V1SecurityContext( + privileged=True, + ), + ) + pod = client.V1Pod( + metadata=client.V1ObjectMeta( + generate_name=name+'-', + namespace=namespace, + labels={"app": "roman"}, + ), + spec=client.V1PodSpec( + containers=[container], + volumes=volumes, + restart_policy="Never", + ) + ) + v1 = client.CoreV1Api() + res = v1.create_namespaced_pod(namespace, pod) + return res diff --git a/apluslms_roman/utils/path_mapping.py b/apluslms_roman/utils/path_mapping.py new file mode 100644 index 0000000..3d447c4 --- /dev/null +++ b/apluslms_roman/utils/path_mapping.py @@ -0,0 +1,71 @@ +from pathlib import PurePosixPath +import json +import logging +import re +from os import environ + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +json_re = re.compile(r'^(?:["[{]|(?:-?[1-9]\d*(?:\.\d+)?|null|true|false)$)') + + +def get_host_path(original, mapping): + ret = original + orig_path = PurePosixPath(original) + for k, v in mapping.items(): + try: + logger.debug("Mapping:{}:{}".format(k, v)) + relative_path = orig_path.relative_to(k) + ret = PurePosixPath(v).joinpath(relative_path) + return str(ret) + except ValueError: + logger.critical("Error when composing new path!") + pass + return str(ret) + + +def get_pair_form_env(key, json_str, read_key=None): + if key == read_key: + try: + ret = json.loads(json_str) + if isinstance(ret, dict): + return ret + except json.decoder.JSONDecodeError: + logger.critical("Error, check your json string") + return json_str + + +def env_value_to_dict(json_str): + if json_re.match(json_str): + try: + ret = json.loads(json_str) + return ret + except json.decoder.JSONDecodeError: + logger.critical("Error, check your json string") + return json_str + + +def nest_dict(flat_dict, sep): + ret = {} + for k, v in flat_dict.items(): + key_list = k.split(sep, 1) + if len(key_list) == 2: + root = key_list[0] + if root not in ret: + ret[root] = {} + ret[root][key_list[1]] = v + else: + ret[k] = v + return ret + + +def load_from_env(env_prefix=None, sep=None, decode_json=True): + if decode_json: + decode = lambda s: json.loads(s) if json_re.match(s) is not None else s + else: + decode = lambda s: s + env = {k[len(env_prefix):].lower(): decode(v) for k, v in environ.items() if k.startswith(env_prefix)} + if sep is not None: + env = nest_dict(env, sep) + return env \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4b1ef59..188273f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ apluslms-yamlidator appdirs >=1.4.0, <2 +kubernetes diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/tests/utils/test_kubernetes.py b/tests/utils/test_kubernetes.py new file mode 100644 index 0000000..5c44827 --- /dev/null +++ b/tests/utils/test_kubernetes.py @@ -0,0 +1,39 @@ +import unittest +from apluslms_roman.utils.kubernetes import create_pod +from kubernetes import client, config + +test_cases = [ + { + 'image': 'foo/bar:latest', + 'command': 'ls', + 'environment': {'foo': 'bar'}, + 'name': 'bar', + 'namespace': 'default', + 'mounts': [ + client.V1VolumeMount( + mount_path='/', + name='build-path' + ) + ], + 'volumes': [ + client.V1Volume( + name='build-path', + host_path=client.V1HostPathVolumeSource(path="/build-source") + ) + ], + 'working_dir': '/' + } +] + + +class TestPod(unittest.TestCase): + def test_create_pod(self): + for i, test_case in enumerate(test_cases): + with self.subTest(i=i): + config.load_kube_config() + pod = create_pod(**test_case) + self.assertTrue(pod.metadata.name.startswith(test_case['name'])) + self.assertEqual(pod.metadata.namespace, test_case['namespace']) + self.assertEqual(pod.spec.containers[0].env, [client.V1EnvVar(k, v) for k, v in {'foo': 'bar'}.items()]) + self.assertEqual(pod.spec.containers[0].working_dir, test_case['working_dir']) + self.assertEqual(pod.spec.containers[0].volume_mounts[0].name, test_case['volumes'][0].name) diff --git a/tests/utils/test_path_mapping.py b/tests/utils/test_path_mapping.py new file mode 100644 index 0000000..3ebecb7 --- /dev/null +++ b/tests/utils/test_path_mapping.py @@ -0,0 +1,63 @@ +import unittest +from json import loads, dumps +from apluslms_roman.utils.path_mapping import json_re, env_value_to_dict + +test_case_loadable = ( + 'true', + 'false', + 'null', + '123', + '-123', + '3.14', + '-3.14', + '{"foo": "bar"}', + '[1, 2, 3]', + '"foo bar"' +) + +test_case_not_loadable = ( + "/foobar.py", + "text", + "yes", + "0123123", +) + + +class TestJsonLoadable(unittest.TestCase): + + def test_loadable_not_raise(self): + for i, case in enumerate(test_case_loadable): + with self.subTest(i=i): + loads(case) + + def test_not_loadable_raise(self): + for i, case in enumerate(test_case_not_loadable): + with self.subTest(i=i): + with self.assertRaises(ValueError, msg="Testing:{}".format(case)): + loads(case) + + +class TestJsonRegex(unittest.TestCase): + + def test_loadable_match(self): + for i, case in enumerate(test_case_loadable): + with self.subTest(i=i): + self.assertTrue(json_re.match(case)is not None, msg="Testing:{}".format(case)) + + def test_not_loadable_not_match(self): + for i, case in enumerate(test_case_not_loadable): + with self.subTest(i=i): + self.assertFalse(json_re.match(case) is not None, msg="Testing:{}".format(case)) + + +class TestValueDict(unittest.TestCase): + + def test_loadable_type(self): + for i, case in enumerate(test_case_loadable): + with self.subTest(i=i): + self.assertEqual(env_value_to_dict(case), loads(case), msg="Testing:{}".format(case)) + + def test_not_loadable_type(self): + for i, case in enumerate(test_case_not_loadable): + with self.subTest(i=i): + self.assertEqual(env_value_to_dict(case), case, msg="Testing:{}".format(case))