diff --git a/python/examples/pipeline/imdb_model_ensemble/config.yml b/python/examples/pipeline/imdb_model_ensemble/config.yml index 13a85d068e2a1bbea5caa2a552ce4f26cb3b0316..3f0b1bb8d4eedb073fa5014eb20e1a170f0d811b 100644 --- a/python/examples/pipeline/imdb_model_ensemble/config.yml +++ b/python/examples/pipeline/imdb_model_ensemble/config.yml @@ -6,3 +6,5 @@ dag: client_type: brpc retry: 1 use_profile: false + tracer: + interval_s: 10 diff --git a/python/examples/pipeline/imdb_model_ensemble/test_pipeline_server.py b/python/examples/pipeline/imdb_model_ensemble/test_pipeline_server.py index a9e5b16ccd1d9861b41d60c1bf1b641c688f59d5..515fd352483e1a3ced57014e11d4faff2a6faddb 100644 --- a/python/examples/pipeline/imdb_model_ensemble/test_pipeline_server.py +++ b/python/examples/pipeline/imdb_model_ensemble/test_pipeline_server.py @@ -12,20 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=doc-string-missing - -import logging -logging.basicConfig( - format="[%(process)d](%(threadName)s) %(levelname)s %(asctime)s [%(filename)s:%(lineno)d] %(message)s", - level=logging.INFO) - +import paddle_serving_server.pipeline as pipeline from paddle_serving_server.pipeline import Op, RequestOp, ResponseOp from paddle_serving_server.pipeline import PipelineServer from paddle_serving_server.pipeline.proto import pipeline_service_pb2 from paddle_serving_server.pipeline.channel import ChannelDataEcode import numpy as np from paddle_serving_app.reader import IMDBDataset +import logging _LOGGER = logging.getLogger() +console_handler = pipeline.logger.StreamHandler() +console_handler.setLevel(logging.INFO) +console_handler.setFormatter( + logging.Formatter( + "%(levelname)s %(asctime)s [%(filename)s:%(lineno)d] %(message)s")) +_LOGGER.addHandler(console_handler) class ImdbRequestOp(RequestOp): diff --git a/python/pipeline/analyse.py b/python/pipeline/analyse.py index b37484a3f31106ecee5c80d3f9cc6d81d6a25fc6..11a68b272e88fc6575d48b43ff0ea400702e45db 100644 --- a/python/pipeline/analyse.py +++ b/python/pipeline/analyse.py @@ -17,7 +17,7 @@ import copy import re import logging -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger("pipeline.analyse") class Analyst(object): diff --git a/python/pipeline/channel.py b/python/pipeline/channel.py index e6fa852e172ce31c99b5923931c75794cdda4412..08c8936e50465b2e2bb9ec5e025de117af71e061 100644 --- a/python/pipeline/channel.py +++ b/python/pipeline/channel.py @@ -29,7 +29,7 @@ import enum import os import copy -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger("pipeline.channel") class ChannelDataEcode(enum.Enum): diff --git a/python/pipeline/dag.py b/python/pipeline/dag.py index 21a579b81285496cd49bece54b99c367bfcb75cf..3a9e8c6fc534098287589d30c3fcb9d19f697453 100644 --- a/python/pipeline/dag.py +++ b/python/pipeline/dag.py @@ -32,7 +32,7 @@ from .profiler import TimeProfiler, PerformanceTracer from .util import NameGenerator from .proto import pipeline_service_pb2 -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger("pipeline.dag") class DAGExecutor(object): diff --git a/python/pipeline/logger.py b/python/pipeline/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..fd86b58cc9042b88dbaaf71ca2e132de330be8f1 --- /dev/null +++ b/python/pipeline/logger.py @@ -0,0 +1,90 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import logging.handlers +import os + + +class SectionLevelFilter(object): + def __init__(self, levels): + self._levels = levels + + def filter(self, logRecord): + return logRecord.levelno in self._levels + + +class OutOfMouduleFilter(object): + def __init__(self, out_names): + self._out_names = out_names + + def filter(self, logRecord): + return logRecord.name not in self._out_names + + +class OutOfMouduleAndSectionLevelFilter(object): + def __init__(self, out_names, levels): + self._out_names = out_names + self._levels = levels + + def filter(self, logRecord): + if logRecord.name in self._out_names: + return False + return logRecord.levelno in self._levels + + +class StreamHandler(logging.StreamHandler): + def __init__(self, *args, **kwargs): + super(StreamHandler, self).__init__(*args, **kwargs) + self.addFilter(OutOfMouduleFilter(["pipeline.profiler"])) + + +log_dir = "PipelineServingLogs" +if not os.path.exists(log_dir): + os.makedirs(log_dir) + +# root logger +_LOGGER = logging.getLogger() +_LOGGER.setLevel(logging.DEBUG) + +formatter = logging.Formatter( + "%(levelname)s %(asctime)s [%(filename)s:%(lineno)d] %(message)s") +# info and warn +file_info = logging.handlers.RotatingFileHandler( + os.path.join(log_dir, "INFO.log")) +file_info.addFilter(OutOfMouduleFilter(["pipeline.profiler"])) +file_info.addFilter(SectionLevelFilter([logging.INFO, logging.WARNING])) +file_info.setFormatter(formatter) + +# err and critical +file_err = logging.handlers.RotatingFileHandler( + os.path.join(log_dir, "ERROR.log")) +file_err.addFilter(OutOfMouduleFilter(["pipeline.profiler"])) +file_err.setLevel(logging.ERROR) +file_err.setFormatter(formatter) + +_LOGGER.addHandler(file_info) +_LOGGER.addHandler(file_err) + +# tracer logger +_TRACER = logging.getLogger("pipeline.profiler") +_TRACER.setLevel(logging.INFO) +_TRACER.addFilter(logging.Filter("pipeline.profiler")) + +# tracer +tracer_formatter = logging.Formatter("%(asctime)s %(message)s") +file_trace = logging.handlers.RotatingFileHandler( + os.path.join(log_dir, "TRACE.log")) +file_trace.setFormatter(tracer_formatter) +_TRACER.addHandler(file_trace) diff --git a/python/pipeline/operator.py b/python/pipeline/operator.py index 2b6a992704734ea11299280e2424a40e5355d7fe..323b4865cf1f4529a50e85b94723ecc09c0bd0b3 100644 --- a/python/pipeline/operator.py +++ b/python/pipeline/operator.py @@ -32,7 +32,7 @@ from .channel import (ThreadChannel, ProcessChannel, ChannelDataEcode, from .util import NameGenerator from .profiler import UnsafeTimeProfiler as TimeProfiler -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger("pipeline.operator") _op_name_gen = NameGenerator("Op") diff --git a/python/pipeline/pipeline_client.py b/python/pipeline/pipeline_client.py index 405c3625ddc55e2965235a26906f08ea7fbfcf37..0b4226dafbddc95bedf9d638c73e0600e78a4184 100644 --- a/python/pipeline/pipeline_client.py +++ b/python/pipeline/pipeline_client.py @@ -22,7 +22,7 @@ from .channel import ChannelDataEcode from .proto import pipeline_service_pb2 from .proto import pipeline_service_pb2_grpc -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger("pipeline.pipeline_client") class PipelineClient(object): diff --git a/python/pipeline/pipeline_server.py b/python/pipeline/pipeline_server.py index 74d1f3b83f77296929b6ba036ffd0fa27b5eccdd..d17afde7e88c18a535762d8fbe299fdc4611676f 100644 --- a/python/pipeline/pipeline_server.py +++ b/python/pipeline/pipeline_server.py @@ -26,7 +26,7 @@ from .proto import pipeline_service_pb2_grpc from .operator import ResponseOp from .dag import DAGExecutor -_LOGGER = logging.getLogger() +_LOGGER = logging.getLogger("pipeline.pipeline_server") class PipelineServicer(pipeline_service_pb2_grpc.PipelineServiceServicer): diff --git a/python/pipeline/profiler.py b/python/pipeline/profiler.py index 81c1187ce364b60a0c6ea682ccc2ce9d83ebbe76..6d7f2b63f0a22b00af712c913922c01154c1b008 100644 --- a/python/pipeline/profiler.py +++ b/python/pipeline/profiler.py @@ -27,7 +27,7 @@ import time import threading import multiprocessing -_TRACER = logging.getLogger("tracer") +_TRACER = logging.getLogger("pipeline.profiler") class PerformanceTracer(object):