Commit 7a37eb2b authored by unknown's avatar unknown
Browse files

Inconsistency, logging fixes to master 1.6.59

parent 2ddb3a73
......@@ -90,8 +90,9 @@ def activate_task_worker(task_type):
for task_param in task_params:
log_dict = {'task': 'check_task_params', 'data': {'task_param_id': task_param['id'], 'task_type': task_type, 'is_equal': task_param['id'] == task_type}}
logging.getLogger(INFO_LOGGER).info("Activates task worker", extra=log_dict)
# Instantiate worker
worker_instance = task_param['worker']()
return worker_instance
if task_param['id'] == task_type:
# Instantiate worker
worker_instance = task_param['worker']()
return worker_instance
return None
......@@ -58,8 +58,8 @@ class TagModelWorker(BaseWorker):
def _generate_loggers(self):
import graypy
info_logger = self.info_logger
error_logger = self.error_logger
info_logger = logging.getLogger(INFO_LOGGER)
error_logger = logging.getLogger(ERROR_LOGGER)
handler = graypy.GELFUDPHandler(os.getenv("GRAYLOG_HOST_NAME", "localhost"), int(os.getenv("GRAYLOG_PORT", 12201)))
info_logger.addHandler(handler)
......
......@@ -144,9 +144,10 @@ class EsDataSample(object):
def _get_positive_samples(self):
sample_size = self.max_positive_sample_size
positive_samples_map = {}
positive_set = set()
# List instead of a set, because in multiindex, documents with the same id repeat
# if they are in a set, it will cause an inconsistency error
positive_set = []
# Initialize sample map
for field in self.fields:
positive_samples_map[field] = []
......@@ -171,7 +172,7 @@ class EsDataSample(object):
try:
# Save sampled doc id
doc_id = str(hit['_id'])
positive_set.add(doc_id)
positive_set.append(doc_id)
for field in self.fields:
# Extract text content for every field
_temp_text = hit['_source']
......@@ -205,7 +206,9 @@ class EsDataSample(object):
def _get_negative_samples(self, positive_set):
negative_samples_map = {}
negative_set = set()
# List instead of a set, because in multiindex, documents with the same id repeat
# if they are in a set, it will cause an inconsistency error
negative_set = []
# Initialize sample map
for field in self.fields:
negative_samples_map[field] = []
......@@ -232,7 +235,7 @@ class EsDataSample(object):
# If used already, continue
continue
# Otherwise, consider as negative sample
negative_set.add(doc_id)
negative_set.append(doc_id)
for field in self.fields:
# Extract text content for every field
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment