# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License import os import copy import argparse from . import constants class BaseConfig(object): def __init__(self, category, config_dict=None): self._category = category self._config_dict = None if config_dict is not None: if isinstance(config_dict, dict): self._config_dict = config_dict else: raise ValueError( "Expected a dictionary. But received: {}".format( config_dict)) # Initialize attributes by the default config config = constants.get_category_default_config(self._category) for field, default_value in config.items(): setattr(self, field, default_value) # Overide attributes by the config_dict if self._config_dict: self.from_dict(self._config_dict) def from_dict(self, config_dict): config = constants.get_category_default_config(self._category) for field in config.keys(): value = config_dict.get(field, constants.NOT_FOUND) # Use the default value if we cannot found the value if value != constants.NOT_FOUND: setattr(self, field, value) def to_dict(self): result_dict = {} config = constants.get_category_default_config(self._category) for field in config.keys(): value = getattr(self, field) result_dict[field] = value for field, value in self.__dict__.items(): if isinstance(value, BaseConfig): result_dict[field] = value.to_dict() return result_dict def __repr__(self): result_dict = self.to_dict() string = "{" for k, v in result_dict.items(): string += "\"%s\":\"%s\"," % (k, v) return string + "}" def __deepcopy__(self, memo): cls = self.__class__ result = cls.__new__(cls) memo[id(self)] = result for k, v in self.__dict__.items(): setattr(result, k, copy.deepcopy(v, memo)) return result class RecomputeConfig(BaseConfig): def __init__(self, config_dict=None): category = constants.RECOMPUTE super(RecomputeConfig, self).__init__(category, config_dict) class AMPConfig(BaseConfig): def __init__(self, config_dict=None): category = constants.AMP super(AMPConfig, self).__init__(category, config_dict) class ShardingConfig(BaseConfig): def __init__(self, config_dict=None): category = constants.SHARDING super(ShardingConfig, self).__init__(category, config_dict) class GradientMergeConfig(BaseConfig): def __init__(self, config_dict=None): category = constants.GRADIENT_MERGE super(GradientMergeConfig, self).__init__(category, config_dict) class QATConfig(BaseConfig): def __init__(self, config_dict=None): category = constants.QAT super(QATConfig, self).__init__(category, config_dict) class TuningConfig(BaseConfig): def __init__(self, config_dict=None): category = constants.TUNING super(TuningConfig, self).__init__(category, config_dict) class Strategy(BaseConfig): """ The `Strategy` object is used to configure the paralleization and optimization beheviors. Args: config (dict|string, optional): If this is None, the default configurations will used. If this is a dictionary, the recognized key-value of it will be used to override the default configurations while other default configurations are left unchanged. If this is a string, it is interpreted as the path to a YAML configuration and will be loaded to override the corresponding default configurations. Examples: .. code-block:: python import paddle from paddle.distributed.fleet import auto strategy = auto.Strategy() sharding = strategy.sharding self.assertEqual(sharding.enabled, False) self.assertEqual(sharding.stage, 1) self.assertEqual(sharding.degree, 8) sharding.enabled = True sharding.stage = 2 sharding.degree = 2 self.assertEqual(sharding.enabled, True) self.assertEqual(sharding.stage, 2) self.assertEqual(sharding.degree, 2) """ def __init__(self, config=None): if config is not None: if isinstance(config, dict): self._config_dict = copy.deepcopy(config) # elif os.path.exists(config): # with open(config, "rb") as yaml_file: # self._config_dict = yaml.load(yaml_file, Loader=yaml.Loader) else: raise ValueError( "Expected a dictionary. But received: {}".format(config)) else: self._config_dict = {} category = constants.BASE super(Strategy, self).__init__(category, self._config_dict) config_dict = self._config_dict.get(constants.RECOMPUTE, None) self.recompute = RecomputeConfig(config_dict) config_dict = self._config_dict.get(constants.AMP, None) self.amp = AMPConfig(config_dict) config_dict = self._config_dict.get(constants.SHARDING, None) self.sharding = ShardingConfig(config_dict) config_dict = self._config_dict.get(constants.GRADIENT_MERGE, None) self.gradient_merge = GradientMergeConfig(config_dict) config_dict = self._config_dict.get(constants.QAT, None) self.qat = QATConfig(config_dict) config_dict = self._config_dict.get(constants.TUNING, None) self.tuning = TuningConfig(config_dict)