add todo

core/dbt/contracts/project.py

@@ -230,6 +230,7 @@ class UserConfig(ExtensibleDbtClassMixin, Replaceable, UserConfigContract):
     use_colors: Optional[bool] = None
     partial_parse: Optional[bool] = None
     printer_width: Optional[int] = None
+    #TODO: add the yaml config options here
 
     def set_values(self, cookie_dir):
         if self.send_anonymous_usage_stats:

core/dbt/flags.py

@@ -18,6 +18,8 @@ WRITE_JSON = None
 PARTIAL_PARSE = None
 USE_COLORS = None
 STORE_FAILURES = None
+LOGGER_MAX_SIZE = None
+LOGGER_BACKUP_COUNT = None
 
 
 def env_set_truthy(key: str) -> Optional[str]:
@@ -56,7 +58,7 @@ MP_CONTEXT = _get_context()
 def reset():
     global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
         USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, \
-        STORE_FAILURES
+        STORE_FAILURES, LOGGER_MAX_SIZE, LOGGER_BACKUP_COUNT
 
     STRICT_MODE = False
     FULL_REFRESH = False
@@ -69,12 +71,14 @@ def reset():
     MP_CONTEXT = _get_context()
     USE_COLORS = True
     STORE_FAILURES = False
+    LOGGER_MAX_SIZE = 10 # default 10mb
+    LOGGER_BACKUP_COUNT = 5
 
 
 def set_from_args(args):
     global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
         USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, \
-        STORE_FAILURES
+        STORE_FAILURES, LOGGER_MAX_SIZE, LOGGER_BACKUP_COUNT
 
     USE_CACHE = getattr(args, 'use_cache', USE_CACHE)
 
@@ -98,7 +102,8 @@ def set_from_args(args):
     if use_colors_override is not None:
         USE_COLORS = use_colors_override
 
-    STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES)
+    LOGGER_MAX_SIZE = getattr(args, 'logger_max_size', LOGGER_MAX_SIZE)
+    LOGGER_BACKUP_COUNT = getattr(args, 'logger_backup_count', LOGGER_BACKUP_COUNT)
 
 
 # initialize everything to the defaults on module load

core/dbt/logger.py

@@ -389,8 +389,8 @@ class DelayedFileHandler(logbook.RotatingFileHandler, FormatterMixin):
         level=logbook.DEBUG,
         filter=None,
         bubble=True,
-        max_size=10 * 1024 * 1024,  # 10 mb
+        max_size=dbt.flags.LOGGER_MAX_SIZE * 1024 * 1024,  # in megabytes
-        backup_count=5,
+        backup_count=dbt.flags.LOGGER_BACKUP_COUNT,
     ) -> None:
         self.disabled = False
         self._msg_buffer: Optional[List[logbook.LogRecord]] = []
@@ -537,6 +537,12 @@ class LogManager(logbook.NestedSetup):
 
     def set_path(self, path):
         self._file_handler.set_path(path)
+    
+    def set_logger_file_configs(self):
+        """"Refreshes the log file configs with newly defined global flags"""
+        self._file_handler = DelayedFileHandler(
+            max_size=(dbt.flags.LOGGER_MAX_SIZE * 1024 * 1024), 
+            backup_count=dbt.flags.LOGGER_BACKUP_COUNT)
 
     def initialized(self):
         return self._file_handler.initialized

core/dbt/main.py

@@ -1057,6 +1057,26 @@ def parse_args(args, cls=DBTArgumentParser):
         '''
     )
 
+    p.add_argument(
+        '--logger-max-size',
+        default=10,
+        type=int,
+        dest='logger_max_size',
+        help='''
+        Set the max size of individual log files. Currently defaults to 10mb per file.
+        '''
+    )
+
+    p.add_argument(
+        '--logger-backup-count',
+        default=5,
+        type=int,
+        dest='logger_backup_count',
+        help='''
+        Set the max number of individual backup log files generated. Currently defaults to 5 backups.
+        '''
+    )
+
     subs = p.add_subparsers(title="Available sub-commands")
 
     base_subparser = _build_base_subparser()

core/dbt/task/base.py

@@ -68,6 +68,7 @@ class BaseTask(metaclass=ABCMeta):
             log_manager.format_json()
         else:
             log_manager.format_text()
+        log_manager.set_logger_file_configs()
 
     @classmethod
     def from_args(cls, args):