commit d51dce12bdf7f6d6e59ea09282121e42943cc85c Author: Charlie Swanson Date: Fri Oct 14 18:02:33 2016 -0400 Better resmoke logging diff --git a/buildscripts/resmokeconfig/loggers/console.yml b/buildscripts/resmokeconfig/loggers/console.yml index b233de4..787adcb 100644 --- a/buildscripts/resmokeconfig/loggers/console.yml +++ b/buildscripts/resmokeconfig/loggers/console.yml @@ -4,10 +4,18 @@ logging: handlers: - class: logging.StreamHandler fixture: - format: '[%(name)s] %(message)s' + format: '[%(job_num)02d:%(name)s] %(message)s' handlers: - class: logging.StreamHandler + - class: logging.MultiFileHandler + filename_pattern: 'fixture%(job_num)02d.log' + log_dir: fixture_logs + mode: w tests: - format: '[%(name)s] %(asctime)s %(message)s' + format: '[%(job_num)02d:%(name)s] %(asctime)s %(message)s' handlers: - class: logging.StreamHandler + - class: logging.MultiFileHandler + filename_pattern: 'tests%(job_num)02d.log' + log_dir: test_logs + mode: w diff --git a/buildscripts/resmokelib/logging/__init__.py b/buildscripts/resmokelib/logging/__init__.py index 54609ad..7bf7923 100644 --- a/buildscripts/resmokelib/logging/__init__.py +++ b/buildscripts/resmokelib/logging/__init__.py @@ -4,11 +4,9 @@ Extension to the logging package to support buildlogger. from __future__ import absolute_import -# Alias the built-in logging.Logger class for type checking arguments. Those interested in -# constructing a new Logger instance should use the loggers.new_logger() function instead. -from logging import Logger - from . import config from . import buildlogger from . import flush from . import loggers + +from .loggers import LoggerAdapter as Logger diff --git a/buildscripts/resmokelib/logging/config.py b/buildscripts/resmokelib/logging/config.py index c3960bb..79c1d5a 100644 --- a/buildscripts/resmokelib/logging/config.py +++ b/buildscripts/resmokelib/logging/config.py @@ -9,6 +9,7 @@ import sys from . import buildlogger from . import formatters +from . import handlers from . import loggers @@ -123,6 +124,10 @@ def _configure_logger(logger, logger_info): if handler_class == "logging.FileHandler": handler = logging.FileHandler(filename=handler_info["filename"], mode=handler_info.get("mode", "w")) + if handler_class == "logging.MultiFileHandler": + handler = handlers.MultiFileHandler(filename_pattern=handler_info["filename_pattern"], + log_dir=handler_info["log_dir"], + mode=handler_info.get("mode", "w")) elif handler_class == "logging.NullHandler": handler = logging.NullHandler() elif handler_class == "logging.StreamHandler": diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py index 6ede7d38..58b8067 100644 --- a/buildscripts/resmokelib/logging/handlers.py +++ b/buildscripts/resmokelib/logging/handlers.py @@ -7,6 +7,7 @@ from __future__ import absolute_import import json import logging +import os import threading import urllib2 @@ -15,6 +16,64 @@ from ..utils import timer _TIMEOUT_SECS = 10 + +class MultiFileHandler(logging.Handler): + """ + A container of many logging.FileHandlers, takes a filename pattern, and determines which file + to write to based on the log record, creating new logging.FileHandlers as needed. + """ + + def __init__(self, filename_pattern, log_dir='.', mode='a', encoding=None, delay=0): + logging.Handler.__init__(self) + + # Save parameters to be passed to FileHandler constructor later. + self.mode = mode + self.encoding = encoding + self.delay = delay + + if log_dir != '.': + try: + os.mkdir(log_dir) + except OSError: + pass # The directory already exists. + + self.filename_pattern = log_dir + os.sep + filename_pattern + + # maps file name to FileHandler instance. Dynamically created as needed. + self.file_handlers = {} + + def emit(self, record): + """ + Emits a record. + + Determine which file this record should go to by formatting the filename_pattern with the + attributes defined on the record. + """ + + filename = self.filename_pattern % record.__dict__ + file_handler = self.file_handlers.get(filename, None) + if file_handler is None: + file_handler = logging.FileHandler(filename, self.mode, self.encoding, self.delay) + file_handler.setFormatter(self.formatter) + self.file_handlers[filename] = file_handler + + file_handler.emit(record) + + def flush(self): + """ + Calls flush on all FileHandlers. + """ + for filename in self.file_handlers: + self.file_handlers[filename].flush() + + def close(self): + """ + Calls close on all FileHandlers. + """ + for filename in self.file_handlers: + self.file_handlers[filename].close() + + class BufferedHandler(logging.Handler): """ A handler class that buffers logging records in memory. Whenever diff --git a/buildscripts/resmokelib/logging/loggers.py b/buildscripts/resmokelib/logging/loggers.py index 35f4151..56f409e 100644 --- a/buildscripts/resmokelib/logging/loggers.py +++ b/buildscripts/resmokelib/logging/loggers.py @@ -6,17 +6,47 @@ from __future__ import absolute_import import logging +from .. import utils + EXECUTOR_LOGGER_NAME = "executor" FIXTURE_LOGGER_NAME = "fixture" TESTS_LOGGER_NAME = "tests" -def new_logger(logger_name, parent=None): + +class LoggerAdapter(logging.Logger): """ - Returns a new logging.Logger instance with the specified name. + Based of the logging.LoggerAdapter class, but inherits from logging.Logger so that it can be + used more transparently. + + Allows adding extra attributes to each log record, to be used in the formatter. """ + def __init__(self, logger_name, level=logging.NOTSET, extra=None): + logging.Logger.__init__(self, logger_name, level) + self.extra = utils.default_if_none(extra, {}).copy() + + def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None): + """ + Adds attributes from 'self.extra' to the record. + """ + record = logging.Logger.makeRecord( + self, name, level, fn, lno, msg, args, exc_info, func, extra) + for key in self.extra: + record.__dict__[key] = self.extra[key] + return record + + +def new_logger(logger_name, parent=None, extra=None): + """ + Returns an instance of 'LoggerAdapter' with the given extra attributes, setting the parent of + the logger if specified. + """ + if parent is not None: + inherited_extra = parent.extra.copy() + inherited_extra.update(utils.default_if_none(extra, {})) + extra = inherited_extra # Set up the logger to handle all messages it receives. - logger = logging.Logger(logger_name, level=logging.DEBUG) + logger = LoggerAdapter(logger_name, level=logging.DEBUG, extra=extra) if parent is not None: logger.parent = parent diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 3628fa0..d35bea5 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -230,8 +230,14 @@ class TestGroupExecutor(object): fixture_config = self.fixture_config.copy() fixture_class = fixture_config.pop("class") - logger_name = "%s:job%d" % (fixture_class, job_num) - logger = logging.loggers.new_logger(logger_name, parent=logging.loggers.FIXTURE) + logger_name = fixtures.short_name_for_fixture(fixture_class) + extra = { + "job_num": job_num, + "port": "", + } + logger = logging.loggers.new_logger(logger_name, + parent=logging.loggers.FIXTURE, + extra=extra) logging.config.apply_buildlogger_global_handler(logger, self.logging_config, build_id=build_id, @@ -250,7 +256,7 @@ class TestGroupExecutor(object): behavior_config = behavior_config.copy() behavior_class = behavior_config.pop("class") - logger_name = "%s:job%d" % (behavior_class, job_num) + logger_name = "%02d:%s" % (job_num, behavior_class) logger = logging.loggers.new_logger(logger_name, parent=self.logger) behavior = _hooks.make_custom_behavior(behavior_class, logger, @@ -270,8 +276,9 @@ class TestGroupExecutor(object): fixture = self._make_fixture(job_num, build_id, build_config) hooks = self._make_hooks(job_num, fixture) - logger_name = "%s:job%d" % (self.logger.name, job_num) - logger = logging.loggers.new_logger(logger_name, parent=self.logger) + logger_name = "%02d:%s:" % (job_num, self.logger.name) + logger = logging.loggers.new_logger( + logger_name, parent=self.logger, extra={"job_num": job_num}) if build_id is not None: endpoint = logging.buildlogger.APPEND_GLOBAL_LOGS_ENDPOINT % {"build_id": build_id} diff --git a/buildscripts/resmokelib/testing/fixtures/__init__.py b/buildscripts/resmokelib/testing/fixtures/__init__.py index d68a669..39fc0b4 100644 --- a/buildscripts/resmokelib/testing/fixtures/__init__.py +++ b/buildscripts/resmokelib/testing/fixtures/__init__.py @@ -22,6 +22,13 @@ _FIXTURES = { } +def short_name_for_fixture(class_name): + """ + Returns a shortened name of the fixture, to be used as the name of the logger for that fixture. + """ + return _FIXTURES[class_name].SHORT_NAME + + def make_fixture(class_name, *args, **kwargs): """ Factory function for creating Fixture instances. diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index 8921aa1..a2f332f 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -17,6 +17,8 @@ class Fixture(object): Base class for all fixtures. """ + SHORT_NAME = "F" + def __init__(self, logger, job_num): """ Initializes the fixtures with a logger instance. diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py index 2bcd3c5..82077fd 100644 --- a/buildscripts/resmokelib/testing/fixtures/masterslave.py +++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py @@ -22,6 +22,8 @@ class MasterSlaveFixture(interface.ReplFixture): run against. """ + SHORT_NAME = "m/s" + def __init__(self, logger, job_num, @@ -139,7 +141,7 @@ class MasterSlaveFixture(interface.ReplFixture): master of a master-slave deployment. """ - logger_name = "%s:master" % (self.logger.name) + logger_name = "M" mongod_logger = logging.loggers.new_logger(logger_name, parent=self.logger) mongod_options = self.mongod_options.copy() @@ -154,7 +156,7 @@ class MasterSlaveFixture(interface.ReplFixture): slave of a master-slave deployment. """ - logger_name = "%s:slave" % (self.logger.name) + logger_name = "S" mongod_logger = logging.loggers.new_logger(logger_name, parent=self.logger) mongod_options = self.mongod_options.copy() diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index 69e82c7..69c0d98 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -21,6 +21,8 @@ class ReplicaSetFixture(interface.ReplFixture): Fixture which provides JSTests with a replica set to run against. """ + SHORT_NAME = "rs" + def __init__(self, logger, job_num, @@ -217,12 +219,12 @@ class ReplicaSetFixture(interface.ReplFixture): """ if index == 0: - logger_name = "%s:primary" % (self.logger.name) + logger_name = "%s:P" % (self.logger.name) elif index == self.initial_sync_node_idx: - logger_name = "%s:initsync" % (self.logger.name) + logger_name = "%s:IS" % (self.logger.name) else: suffix = str(index - 1) if self.num_nodes > 2 else "" - logger_name = "%s:secondary%s" % (self.logger.name, suffix) + logger_name = "%s:S%s" % (self.logger.name, suffix) return logging.loggers.new_logger(logger_name, parent=self.logger) diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index 67f0bbb..15f237a 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -27,6 +27,8 @@ class ShardedClusterFixture(interface.Fixture): against. """ + SHORT_NAME = "cluster" + _CONFIGSVR_REPLSET_NAME = "config-rs" def __init__(self, @@ -176,7 +178,7 @@ class ShardedClusterFixture(interface.Fixture): the config server of a sharded cluster. """ - logger_name = "%s:configsvr" % (self.logger.name) + logger_name = "%s:cfg" % (self.logger.name) mongod_logger = logging.loggers.new_logger(logger_name, parent=self.logger) mongod_options = copy.deepcopy(self.mongod_options) @@ -219,7 +221,7 @@ class ShardedClusterFixture(interface.Fixture): a sharded cluster. """ - logger_name = "%s:mongos" % (self.logger.name) + logger_name = "%s:s" % (self.logger.name) mongos_logger = logging.loggers.new_logger(logger_name, parent=self.logger) mongos_options = copy.deepcopy(self.mongos_options) @@ -258,6 +260,8 @@ class _MongoSFixture(interface.Fixture): Fixture which provides JSTests with a mongos to connect to. """ + SHORT_NAME = "s" + def __init__(self, logger, job_num, @@ -284,6 +288,7 @@ class _MongoSFixture(interface.Fixture): try: self.logger.info("Starting mongos on port %d...\n%s", self.port, mongos.as_command()) mongos.start() + self.logger.extra["port"] = self.port self.logger.info("mongos started on port %d with pid %d.", self.port, mongos.pid) except: self.logger.exception("Failed to start mongos on port %d.", self.port) diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 5ee70d3..486ceb3 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -25,6 +25,8 @@ class MongoDFixture(interface.Fixture): against. """ + SHORT_NAME = "md" + AWAIT_READY_TIMEOUT_SECS = 300 def __init__(self, @@ -79,6 +81,7 @@ class MongoDFixture(interface.Fixture): try: self.logger.info("Starting mongod on port %d...\n%s", self.port, mongod.as_command()) mongod.start() + self.logger.extra["port"] = self.port self.logger.info("mongod started on port %d with pid %d.", self.port, mongod.pid) except: self.logger.exception("Failed to start mongod on port %d.", self.port) diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index 2ef1883..fd36ad0 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -118,8 +118,11 @@ class TestReport(unittest.TestResult): test_info.url_endpoint) # Set up the test-specific logger. + test.logger.extra["job_num"] = self.logger.extra["job_num"] + test.logger.extra["test_name"] = test.short_name() logger_name = "%s:%s" % (test.logger.name, test.short_name()) - logger = logging.loggers.new_logger(logger_name, parent=test.logger) + extra = {"test_name": test.short_name()} + logger = logging.loggers.new_logger(logger_name, parent=test.logger, extra=extra) logging.config.apply_buildlogger_test_handler(logger, self.logging_config, build_id=self.build_id,