From 6a35ad107c258cda50063c91334acd5783a672be Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 01:13:49 +0800 Subject: [PATCH 1/9] fix: in py3.7, as grey typo is not only a warning but error --- image_match/goldberg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/image_match/goldberg.py b/image_match/goldberg.py index da93ff8..e053b1c 100644 --- a/image_match/goldberg.py +++ b/image_match/goldberg.py @@ -236,14 +236,14 @@ def preprocess_image(image_or_path, bytestream=False, handle_mpo=False): return rgb2gray(np.asarray(img, dtype=np.uint8)) elif type(image_or_path) in string_types or \ type(image_or_path) is text_type: - return imread(image_or_path, as_grey=True) + return imread(image_or_path, as_gray=True) elif type(image_or_path) is bytes: try: img = Image.open(image_or_path) arr = np.array(img.convert('RGB')) except IOError: # try again due to PIL weirdness - return imread(image_or_path, as_grey=True) + return imread(image_or_path, as_gray=True) if handle_mpo: # take the first images from the MPO if arr.shape == (2,) and isinstance(arr[1].tolist(), MpoImageFile): From fec2b08fbf81599abbf30cad3a0626de9fca8c8c Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 01:21:55 +0800 Subject: [PATCH 2/9] feat: follow the latest mapping definition for es7.8 --- tests/test_elasticsearch_driver.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_elasticsearch_driver.py b/tests/test_elasticsearch_driver.py index 2a6daaa..63a12da 100644 --- a/tests/test_elasticsearch_driver.py +++ b/tests/test_elasticsearch_driver.py @@ -20,15 +20,16 @@ DOC_TYPE = 'image' MAPPINGS = { "mappings": { - DOC_TYPE: { - "dynamic": True, - "properties": { - "metadata": { - "type": "object", - "dynamic": True, + "properties": { + DOC_TYPE: { + "type": "nested", + "properties": { + "metadata": { + "type": "nested", "properties": { - "tenant_id": { "type": "keyword" } + "tenant_id": { "type": "keyword" } } + } } } } From 4814df89c2a2b67634d9e2a34ea4273416968b19 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:32:05 +0800 Subject: [PATCH 3/9] feat: doc type is deprecated on 6.x --- image_match/elasticsearch_driver.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/image_match/elasticsearch_driver.py b/image_match/elasticsearch_driver.py index 1f105cc..a7f8857 100644 --- a/image_match/elasticsearch_driver.py +++ b/image_match/elasticsearch_driver.py @@ -53,24 +53,23 @@ def search_single_record(self, rec, pre_filter=None): rec.pop('metadata') # build the 'should' list - should = [{'term': {word: rec[word]}} for word in rec] + should = [{'term': {'{}.{}'.format(self.doc_type, word): rec[word]}} for word in rec] body = { 'query': { 'bool': {'should': should} }, - '_source': {'excludes': ['simple_word_*']} + '_source': {'excludes': ['{}.simple_word_*'.format(self.doc_type)]} } if pre_filter is not None: body['query']['bool']['filter'] = pre_filter res = self.es.search(index=self.index, - doc_type=self.doc_type, body=body, size=self.size, timeout=self.timeout)['hits']['hits'] - sigs = np.array([x['_source']['signature'] for x in res]) + sigs = np.array([x['_source'][self.doc_type]['signature'] for x in res]) if sigs.size == 0: return [] @@ -79,8 +78,8 @@ def search_single_record(self, rec, pre_filter=None): formatted_res = [{'id': x['_id'], 'score': x['_score'], - 'metadata': x['_source'].get('metadata'), - 'path': x['_source'].get('url', x['_source'].get('path'))} + 'metadata': x['_source'][self.doc_type].get('metadata'), + 'path': x['_source'][self.doc_type].get('url', x['_source'][self.doc_type].get('path'))} for x in res] for i, row in enumerate(formatted_res): @@ -91,7 +90,7 @@ def search_single_record(self, rec, pre_filter=None): def insert_single_record(self, rec, refresh_after=False): rec['timestamp'] = datetime.now() - self.es.index(index=self.index, doc_type=self.doc_type, body=rec, refresh=refresh_after) + self.es.index(index=self.index, body={ self.doc_type: rec }, refresh=refresh_after) def delete_duplicates(self, path): """Delete all but one entries in elasticsearch whose `path` value is equivalent to that of path. @@ -101,11 +100,11 @@ def delete_duplicates(self, path): matching_paths = [item['_id'] for item in self.es.search(body={'query': {'match': - {'path': path} + {'{}.path'.format(self.doc_type): path} } }, index=self.index)['hits']['hits'] - if item['_source']['path'] == path] + if item['_source'][self.doc_type]['path'] == path] if len(matching_paths) > 0: for id_tag in matching_paths[1:]: - self.es.delete(index=self.index, doc_type=self.doc_type, id=id_tag) + self.es.delete(index=self.index, id=id_tag) From 8164d6b683ba96329efca2594d9b38e52f336b82 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:34:49 +0800 Subject: [PATCH 4/9] feat: migrate the query syntax to es 7.8 --- image_match/signature_database_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image_match/signature_database_base.py b/image_match/signature_database_base.py index 42eadd0..402791f 100644 --- a/image_match/signature_database_base.py +++ b/image_match/signature_database_base.py @@ -52,7 +52,7 @@ def search_single_record(self, rec, pre_filter=None): before applying the matching strategy For example: - { "term": { "metadata.category": "art" } } + { "term": { "image.metadata.category": "art" } } Returns: a formatted list of dicts representing matches. From 17ccf82978fb714f193a83764ff92d4ea288ba36 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:37:11 +0800 Subject: [PATCH 5/9] feat: migrate the mapping syntax to latest version and make sure the type is keyword --- tests/test_elasticsearch_driver.py | 25 ++++++++++-------- ...elasticsearch_driver_metadata_as_nested.py | 26 ++++++++++--------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/tests/test_elasticsearch_driver.py b/tests/test_elasticsearch_driver.py index 63a12da..96aeee7 100644 --- a/tests/test_elasticsearch_driver.py +++ b/tests/test_elasticsearch_driver.py @@ -19,21 +19,24 @@ INDEX_NAME = 'test_environment_{}'.format(hashlib.md5(os.urandom(128)).hexdigest()[:12]) DOC_TYPE = 'image' MAPPINGS = { - "mappings": { - "properties": { - DOC_TYPE: { - "type": "nested", + "mappings": { "properties": { - "metadata": { - "type": "nested", - "properties": { - "tenant_id": { "type": "keyword" } + DOC_TYPE: { + "properties": { + "path": { + "type": "keyword" + }, + "metadata": { + "properties": { + "tenant_id": { + "type": "keyword", + } + } + } + } } - } } - } } - } } diff --git a/tests/test_elasticsearch_driver_metadata_as_nested.py b/tests/test_elasticsearch_driver_metadata_as_nested.py index 3c37c7c..a768f7b 100644 --- a/tests/test_elasticsearch_driver_metadata_as_nested.py +++ b/tests/test_elasticsearch_driver_metadata_as_nested.py @@ -19,21 +19,23 @@ INDEX_NAME = 'test_environment_{}'.format(hashlib.md5(os.urandom(128)).hexdigest()[:12]) DOC_TYPE = 'image' MAPPINGS = { - "mappings": { - DOC_TYPE: { - "dynamic": True, - "properties": { - "metadata": { - "type": "nested", - "dynamic": True, - "properties": { - "tenant_id": { "type": "keyword" }, - "project_id": { "type": "keyword" } + "mappings": { + "properties": { + DOC_TYPE: { + "properties": { + "path": { + "type": "keyword" + }, + "metadata": { + "properties": { + "tenant_id": { "type": "keyword" }, + "project_id": { "type": "keyword" } + } + } + } } } - } } - } } From a941990d2540ca43568b5d1558b26f0e999644e7 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:39:33 +0800 Subject: [PATCH 6/9] fix: the error message has been changed in newer elasticsearch --- tests/test_elasticsearch_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elasticsearch_driver.py b/tests/test_elasticsearch_driver.py index 96aeee7..7338694 100644 --- a/tests/test_elasticsearch_driver.py +++ b/tests/test_elasticsearch_driver.py @@ -50,7 +50,7 @@ def setup_index(request, index_name): try: es.indices.create(index=index_name, body=MAPPINGS) except RequestError as e: - if e.error == u'index_already_exists_exception': + if e.error == u'resource_already_exists_exception': es.indices.delete(index_name) else: raise From ffb2c6273e65e1ef260e8f3e2815387eb80c6cea Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:40:18 +0800 Subject: [PATCH 7/9] test: update search filter syntax --- tests/test_elasticsearch_driver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_elasticsearch_driver.py b/tests/test_elasticsearch_driver.py index 7338694..2725296 100644 --- a/tests/test_elasticsearch_driver.py +++ b/tests/test_elasticsearch_driver.py @@ -193,15 +193,15 @@ def test_lookup_with_filter_by_metadata(ses): ) ses.add_image('test2.jpg', metadata=metadata2, refresh_after=True) - r = ses.search_image('test1.jpg', pre_filter={"term": {"metadata.tenant_id": "foo"}}) + r = ses.search_image('test1.jpg', pre_filter={"term": {'{}.metadata.tenant_id'.format(DOC_TYPE): "foo"}}) assert len(r) == 1 assert r[0]['metadata'] == metadata - r = ses.search_image('test1.jpg', pre_filter={"term": {"metadata.tenant_id": "bar-2"}}) + r = ses.search_image('test1.jpg', pre_filter={"term": {'{}.metadata.tenant_id'.format(DOC_TYPE): "bar-2"}}) assert len(r) == 1 assert r[0]['metadata'] == metadata2 - r = ses.search_image('test1.jpg', pre_filter={"term": {"metadata.tenant_id": "bar-3"}}) + r = ses.search_image('test1.jpg', pre_filter={"term": {'{}.metadata.tenant_id'.format(DOC_TYPE): "bar-3"}}) assert len(r) == 0 From b03c7b4c9de975cbf6ee4fee68615b51d8ab73fc Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:40:59 +0800 Subject: [PATCH 8/9] test: update search in multi filter query --- ...t_elasticsearch_driver_metadata_as_nested.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tests/test_elasticsearch_driver_metadata_as_nested.py b/tests/test_elasticsearch_driver_metadata_as_nested.py index a768f7b..bbf4ede 100644 --- a/tests/test_elasticsearch_driver_metadata_as_nested.py +++ b/tests/test_elasticsearch_driver_metadata_as_nested.py @@ -124,16 +124,7 @@ def _metadata(tenant_id, project_id): ) def _nested_filter(tenant_id, project_id): - return { - "nested" : { - "path" : "metadata", - "query" : { - "bool" : { - "must" : [ - {"term": {"metadata.tenant_id": tenant_id}}, - {"term": {"metadata.project_id": project_id}} - ] - } - } - } - } \ No newline at end of file + return [ + {"term": {"image.metadata.tenant_id": tenant_id}}, + {"term": {"image.metadata.project_id": project_id}} + ] \ No newline at end of file From dfb126ab9b8a6258697d773bded4f5e979870447 Mon Sep 17 00:00:00 2001 From: Calvin Huang Date: Sun, 28 Jun 2020 15:41:37 +0800 Subject: [PATCH 9/9] chore: upgrade depended elasticsearch version in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e05c1f1..29c97db 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ def find_version(*file_paths): ], install_requires=[ 'scikit-image>=0.14', - 'elasticsearch>=5.0.0,<6.0.0', + 'elasticsearch>=7.0.0,<8.0.0', 'six>=1.11.0', ], tests_require=tests_require,