From 6a7e3039f7e527cf28ed533118fe1cefc3eb7504 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 13:07:19 +0800 Subject: [PATCH 1/5] fix(doc_reducer_router): fix reduce error --- gnes/router/reduce/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnes/router/reduce/document.py b/gnes/router/reduce/document.py index 2efad6dd..cc08f1d6 100644 --- a/gnes/router/reduce/document.py +++ b/gnes/router/reduce/document.py @@ -9,7 +9,7 @@ def apply(self, msg: 'gnes_pb2.Message', accum_msgs: List['gnes_pb2.Message'], * final_docs = [] for idx, r in enumerate(msg.response.search.topk_results): # get result from all shards, some may return None, we only take the first non-None doc - final_docs.append([r for m in accum_msgs if + final_docs.append([m for m in accum_msgs if m.response.search.topk_results[idx].doc.WhichOneof('raw_data') is not None][0]) # resort all doc result as the doc_weight has been applied From 260b0f14dcd4e03693725e4987014bf141d534b0 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 13:10:37 +0800 Subject: [PATCH 2/5] fix error ... --- gnes/router/reduce/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gnes/router/reduce/document.py b/gnes/router/reduce/document.py index cc08f1d6..b1251243 100644 --- a/gnes/router/reduce/document.py +++ b/gnes/router/reduce/document.py @@ -9,7 +9,7 @@ def apply(self, msg: 'gnes_pb2.Message', accum_msgs: List['gnes_pb2.Message'], * final_docs = [] for idx, r in enumerate(msg.response.search.topk_results): # get result from all shards, some may return None, we only take the first non-None doc - final_docs.append([m for m in accum_msgs if + final_docs.append([m.response.search.topk_results[idx].doc for m in accum_msgs if m.response.search.topk_results[idx].doc.WhichOneof('raw_data') is not None][0]) # resort all doc result as the doc_weight has been applied From 506b06d6273c6f88c3b33f27fd6a332a28e6646a Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 13:23:03 +0800 Subject: [PATCH 3/5] unable profiling in unittest --- tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 2b4a1e35..dab044ea 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -48,7 +48,7 @@ def line2pb_doc(line: str, doc_id: int = 0, deliminator: str = r'[.。！？!?]+ 'VGG_MODEL': '/', 'RESNET_MODEL': '/', 'INCEPTION_MODEL': '/', - 'GNES_PROFILING': 1 + 'GNES_PROFILING': 0 }, 'idc-165': { 'BERT_CI_PORT': 7125, @@ -63,7 +63,7 @@ def line2pb_doc(line: str, doc_id: int = 0, deliminator: str = r'[.。！？!?]+ 'VGG_MODEL': '/ext_data/image_encoder', 'RESNET_MODEL': '/ext_data/image_encoder', 'INCEPTION_MODEL': '/ext_data/image_encoder', - 'GNES_PROFILING': 1 + 'GNES_PROFILING': 0 } } From fb07ff02fbd93cc07c570211bd0d5b6515ea6586 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 13:07:19 +0800 Subject: [PATCH 4/5] fix(doc_reducer_router): fix reduce error --- Dockerfile | 4 +++- docker-push.sh | 2 +- gnes/router/reduce/document.py | 2 +- tests/__init__.py | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f6d5d9fe..96b3615f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,9 @@ FROM dependency as base ADD . ./ -RUN pip --no-cache-dir install .[all] \ +RUN pip --no-cache-dir install -e .[all] \ && rm -rf /tmp/* +# WORKDIR / + ENTRYPOINT ["gnes"] \ No newline at end of file diff --git a/docker-push.sh b/docker-push.sh index a5734042..af1d8cfd 100755 --- a/docker-push.sh +++ b/docker-push.sh @@ -11,7 +11,7 @@ _docker_push() { docker build --rm --target $TARGET -t ${DOCKER_NAMESPACE}/${PROJ_NAME}:${GIT_TAG} . docker push ${DOCKER_NAMESPACE}/${PROJ_NAME}:${GIT_TAG} printf 'done! and to run the container simply do:\n' - printf "\e[1;33mdocker run --entrypoint "/bin/bash" --rm -v /data1/cips/data:/ext_data -it $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG\e[0m\n" + printf "\e[1;33mdocker run --entrypoint "/bin/bash" --rm -v /data/ext_models/ext_models:/ext_data -it $DOCKER_NAMESPACE/$PROJ_NAME:$GIT_TAG\e[0m\n" } _select_namespace() { diff --git a/gnes/router/reduce/document.py b/gnes/router/reduce/document.py index 2efad6dd..245ec3dc 100644 --- a/gnes/router/reduce/document.py +++ b/gnes/router/reduce/document.py @@ -9,7 +9,7 @@ def apply(self, msg: 'gnes_pb2.Message', accum_msgs: List['gnes_pb2.Message'], * final_docs = [] for idx, r in enumerate(msg.response.search.topk_results): # get result from all shards, some may return None, we only take the first non-None doc - final_docs.append([r for m in accum_msgs if + final_docs.append([m.response.search.topk_results[idx] for m in accum_msgs if m.response.search.topk_results[idx].doc.WhichOneof('raw_data') is not None][0]) # resort all doc result as the doc_weight has been applied diff --git a/tests/__init__.py b/tests/__init__.py index 2b4a1e35..dab044ea 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -48,7 +48,7 @@ def line2pb_doc(line: str, doc_id: int = 0, deliminator: str = r'[.。！？!?]+ 'VGG_MODEL': '/', 'RESNET_MODEL': '/', 'INCEPTION_MODEL': '/', - 'GNES_PROFILING': 1 + 'GNES_PROFILING': 0 }, 'idc-165': { 'BERT_CI_PORT': 7125, @@ -63,7 +63,7 @@ def line2pb_doc(line: str, doc_id: int = 0, deliminator: str = r'[.。！？!?]+ 'VGG_MODEL': '/ext_data/image_encoder', 'RESNET_MODEL': '/ext_data/image_encoder', 'INCEPTION_MODEL': '/ext_data/image_encoder', - 'GNES_PROFILING': 1 + 'GNES_PROFILING': 0 } } From d02f320daceeeb87492e748e960f323856d093a1 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 17 Jul 2019 13:48:47 +0800 Subject: [PATCH 5/5] revert docker file --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96b3615f..32992888 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,9 +13,9 @@ FROM dependency as base ADD . ./ -RUN pip --no-cache-dir install -e .[all] \ +RUN pip --no-cache-dir install .[all] \ && rm -rf /tmp/* -# WORKDIR / +WORKDIR / ENTRYPOINT ["gnes"] \ No newline at end of file