From 60514f5b8d40d4b9742651bbc7678f0095fac947 Mon Sep 17 00:00:00 2001 From: Marcin Kaszynski Date: Tue, 31 Oct 2017 16:53:11 +0100 Subject: [PATCH 1/4] Reformat all code with yapf. --- python/redis_rpc/__init__.py | 47 +++++++++++++++-------------- python/redis_rpc/examples/server.py | 1 - python/redis_rpc/tests/test_base.py | 28 ++++++++--------- python/setup.py | 4 +-- 4 files changed, 39 insertions(+), 41 deletions(-) diff --git a/python/redis_rpc/__init__.py b/python/redis_rpc/__init__.py index 6b898f4..0281c5f 100644 --- a/python/redis_rpc/__init__.py +++ b/python/redis_rpc/__init__.py @@ -6,10 +6,8 @@ from datetime import datetime from uuid import uuid4 - log = logging.getLogger('redis-rpc') - # All timeouts and expiry times are in seconds BLPOP_TIMEOUT = 1 RESPONSE_TIMEOUT = 1 @@ -52,15 +50,13 @@ def escape_for_logs(v): def log_request(func_name, req_bytes, exception, msg): if exception: - log.exception('%s %s %s %s', - func_name, + log.exception('%s %s %s %s', func_name, escape_for_logs(req_bytes.decode()), - escape_for_logs('%s: %s' % (type(exception).__name__, str(exception))), - msg) + escape_for_logs('%s: %s' % (type(exception).__name__, + str(exception))), msg) else: log.info('%s %s - %s', func_name, - escape_for_logs(req_bytes.decode()), - msg) + escape_for_logs(req_bytes.decode()), msg) class Scripts: @@ -77,7 +73,9 @@ def rpush_ex(self, queue, msg, expire): class Client: - def __init__(self, redis, prefix='redis_rpc', + def __init__(self, + redis, + prefix='redis_rpc', request_expire=REQUEST_EXPIRE, blpop_timeout=BLPOP_TIMEOUT, response_timeout=RESPONSE_TIMEOUT): @@ -91,12 +89,12 @@ def __init__(self, redis, prefix='redis_rpc', def call_async(self, func_name, **kwargs): req_id = str(uuid4()) - msg = {'id': req_id, - 'ts': datetime.now().isoformat()} + msg = {'id': req_id, 'ts': datetime.now().isoformat()} msg['kw'] = kwargs - self._scripts.rpush_ex(call_queue_name(self._prefix, func_name), - json.dumps(msg).encode(), self._expire) + self._scripts.rpush_ex( + call_queue_name(self._prefix, func_name), + json.dumps(msg).encode(), self._expire) return req_id @@ -112,7 +110,8 @@ def response(self, func_name, req_id): if now_ts >= deadline_ts: raise RPCTimeout() - wait_time = math.ceil(min(self._blpop_timeout, deadline_ts - now_ts)) + wait_time = math.ceil( + min(self._blpop_timeout, deadline_ts - now_ts)) popped = self._redis.blpop([qn], wait_time) (_, res_bytes) = popped @@ -127,7 +126,9 @@ def call(self, func_name, **kwargs): class Server: - def __init__(self, redis, func_map, + def __init__(self, + redis, + func_map, prefix='redis_rpc', result_expire=RESULT_EXPIRE, blpop_timeout=BLPOP_TIMEOUT): @@ -137,8 +138,10 @@ def __init__(self, redis, func_map, self._expire = result_expire self._blpop_timeout = blpop_timeout self._func_map = func_map - self._queue_map = {call_queue_name(self._prefix, name): (name, func) - for (name, func) in func_map.items()} + self._queue_map = { + call_queue_name(self._prefix, name): (name, func) + for (name, func) in func_map.items() + } self._queue_names = sorted((self._queue_map.keys())) self._call_idx = 0 self._quit = False @@ -156,8 +159,8 @@ def quit(self): self._quit = True def serve_one(self): - popped = self._redis.blpop(rotated(self._queue_names, self._call_idx), - self._blpop_timeout) + popped = self._redis.blpop( + rotated(self._queue_names, self._call_idx), self._blpop_timeout) self._call_idx += 1 if popped is None: return @@ -185,9 +188,9 @@ def serve_one(self): def send_result(self, func_name, req_id, **kwargs): msg = {'ts': datetime.now().isoformat()} msg.update(kwargs) - self._scripts.rpush_ex(response_queue_name(self._prefix, func_name, - req_id), - json.dumps(msg).encode(), self._expire) + self._scripts.rpush_ex( + response_queue_name(self._prefix, func_name, req_id), + json.dumps(msg).encode(), self._expire) def quit_on_signals(self, signals=[signal.SIGTERM, signal.SIGINT]): for s in signals: diff --git a/python/redis_rpc/examples/server.py b/python/redis_rpc/examples/server.py index f07a6fd..0cb57a4 100755 --- a/python/redis_rpc/examples/server.py +++ b/python/redis_rpc/examples/server.py @@ -4,7 +4,6 @@ from redis import StrictRedis from redis_rpc import Server - VALUES = {} diff --git a/python/redis_rpc/tests/test_base.py b/python/redis_rpc/tests/test_base.py index 7da77a4..7fff6e4 100644 --- a/python/redis_rpc/tests/test_base.py +++ b/python/redis_rpc/tests/test_base.py @@ -9,7 +9,6 @@ @contextmanager def rpc_server(redis, func_map, **kwargs): - def server(): rpc = Server(redis, func_map, **kwargs) rpc.serve() @@ -71,8 +70,7 @@ def test_expiry_times(redisdb): def test_server_rotates_queues(): - funcs = {name: lambda: None - for name in ['a', 'b', 'c']} + funcs = {name: lambda: None for name in ['a', 'b', 'c']} mockredis = Mock() mockredis.blpop.return_value = None @@ -81,21 +79,21 @@ def last_call_queues(): srv = Server(mockredis, funcs) srv.serve_one() - assert last_call_queues() == [b'redis_rpc:a:calls', - b'redis_rpc:b:calls', - b'redis_rpc:c:calls'] + assert last_call_queues() == [ + b'redis_rpc:a:calls', b'redis_rpc:b:calls', b'redis_rpc:c:calls' + ] srv.serve_one() - assert last_call_queues() == [b'redis_rpc:b:calls', - b'redis_rpc:c:calls', - b'redis_rpc:a:calls'] + assert last_call_queues() == [ + b'redis_rpc:b:calls', b'redis_rpc:c:calls', b'redis_rpc:a:calls' + ] srv.serve_one() - assert last_call_queues() == [b'redis_rpc:c:calls', - b'redis_rpc:a:calls', - b'redis_rpc:b:calls'] + assert last_call_queues() == [ + b'redis_rpc:c:calls', b'redis_rpc:a:calls', b'redis_rpc:b:calls' + ] srv.serve_one() - assert last_call_queues() == [b'redis_rpc:a:calls', - b'redis_rpc:b:calls', - b'redis_rpc:c:calls'] + assert last_call_queues() == [ + b'redis_rpc:a:calls', b'redis_rpc:b:calls', b'redis_rpc:c:calls' + ] def test_client_timeout(): diff --git a/python/setup.py b/python/setup.py index 5741e0e..e322971 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,5 @@ from setuptools import setup - setup( name='redis-rpc', packages=['redis_rpc'], @@ -12,5 +11,4 @@ setup_requires=['pytest-runner'], install_requires=['redis'], tests_require=['pytest', 'pytest-redis', 'pytest-timeout'], - python_requires=">=3.5" -) + python_requires=">=3.5") From b4386d2065073024e45a2d6120c3200356558fe2 Mon Sep 17 00:00:00 2001 From: Marcin Kaszynski Date: Tue, 31 Oct 2017 16:57:05 +0100 Subject: [PATCH 2/4] Verify code formatting during tests. --- .travis.yml | 4 +++- python/setup.cfg | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 136f9d9..f425039 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,4 +3,6 @@ python: - 3.5 - 3.6 script: - - cd python && python setup.py test + - cd python + - python setup.py test + - pip install yapf && yapf --diff **/*.py diff --git a/python/setup.cfg b/python/setup.cfg index 9af7e6f..6f926bc 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -1,2 +1,5 @@ [aliases] -test=pytest \ No newline at end of file +test=pytest + +[yapf] +based_on_style = pep8 \ No newline at end of file From bca660adbdc5e594da53de320cfc8122bc023c32 Mon Sep 17 00:00:00 2001 From: Marcin Kaszynski Date: Tue, 31 Oct 2017 17:47:37 +0100 Subject: [PATCH 3/4] Check code style as part of `setup.py test`. --- .travis.yml | 1 - python/redis_rpc/tests/test_formatting.py | 10 ++++++++++ python/setup.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 python/redis_rpc/tests/test_formatting.py diff --git a/.travis.yml b/.travis.yml index f425039..5c6e389 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,4 +5,3 @@ python: script: - cd python - python setup.py test - - pip install yapf && yapf --diff **/*.py diff --git a/python/redis_rpc/tests/test_formatting.py b/python/redis_rpc/tests/test_formatting.py new file mode 100644 index 0000000..839e4fd --- /dev/null +++ b/python/redis_rpc/tests/test_formatting.py @@ -0,0 +1,10 @@ +import glob +import os.path +import yapf + +PACKAGE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../..') + + +def test_formatting(): + files = glob.glob("%s/**/*.py" % PACKAGE_DIR, recursive=True) + assert 0 == yapf.main(['arg0-placeholder', '--diff'] + files) diff --git a/python/setup.py b/python/setup.py index e322971..e772e41 100644 --- a/python/setup.py +++ b/python/setup.py @@ -10,5 +10,5 @@ url='https://github.com/marcinkaszynski/redis-rpc', setup_requires=['pytest-runner'], install_requires=['redis'], - tests_require=['pytest', 'pytest-redis', 'pytest-timeout'], + tests_require=['pytest', 'pytest-redis', 'pytest-timeout', 'yapf'], python_requires=">=3.5") From 1e44f27c40175a79d373b3faf09514ba0887a467 Mon Sep 17 00:00:00 2001 From: Marcin Kaszynski Date: Tue, 31 Oct 2017 17:54:32 +0100 Subject: [PATCH 4/4] Make YAPF-detected issues easier to understand. --- python/redis_rpc/tests/test_formatting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/redis_rpc/tests/test_formatting.py b/python/redis_rpc/tests/test_formatting.py index 839e4fd..fc7eccc 100644 --- a/python/redis_rpc/tests/test_formatting.py +++ b/python/redis_rpc/tests/test_formatting.py @@ -7,4 +7,6 @@ def test_formatting(): files = glob.glob("%s/**/*.py" % PACKAGE_DIR, recursive=True) - assert 0 == yapf.main(['arg0-placeholder', '--diff'] + files) + if yapf.main(['arg0-placeholder', '--diff'] + files) != 0: + raise Exception("Please use yapf to format code before submitting. " + "See the diff for details.")