Refine warning configuration

Enable warnings only in unit tests, but exclude DeprecationWarnings in
other modules.

bin/fsck.s3ql

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.fsck
 s3ql.fsck.main(sys.argv[1:])

bin/mkfs.s3ql

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.mkfs
 s3ql.mkfs.main(sys.argv[1:])

bin/mount.s3ql

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.mount
 s3ql.mount.main(sys.argv[1:])

bin/s3ql_oauth_client

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.oauth_client
 s3ql.oauth_client.main(sys.argv[1:])

bin/s3ql_verify

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.verify
 s3ql.verify.main(sys.argv[1:])

bin/s3qladm

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.adm
 s3ql.adm.main(sys.argv[1:])

bin/s3qlcp

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.cp
 s3ql.cp.main(sys.argv[1:])

bin/s3qlctrl

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.ctrl
 s3ql.ctrl.main(sys.argv[1:])

bin/s3qllock

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.lock
 s3ql.lock.main(sys.argv[1:])

bin/s3qlrm

@@ -17,11 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
-
 import s3ql.remove
 s3ql.remove.main(sys.argv[1:])

bin/s3qlstat

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.statfs
 s3ql.statfs.main(sys.argv[1:])

bin/umount.s3ql

@@ -17,10 +17,5 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and
     os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
     sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-# When running from HG repo, enable all warnings
-if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
-    import warnings
-    warnings.simplefilter('default')
-
 import s3ql.umount
 s3ql.umount.main(sys.argv[1:])

src/s3ql/__init__.py

@@ -6,6 +6,21 @@
 This work can be distributed under the terms of the GNU GPLv3.
 '''
 
+# First, enable warnings. This should happen before any imports,
+# so that we can catch warnings emitted by code that runs during
+# the import.
+# Sadly, we cannot simply use $PYTHONWARNINGS, because it
+# does not allow regexes (cf https://bugs.python.org/issue34920)
+import os
+if 'S3QL_ENABLE_WARNINGS' in os.environ:
+    import warnings
+    warnings.resetwarnings()
+    for cat in (DeprecationWarning, PendingDeprecationWarning):
+        warnings.filterwarnings(action='default', category=cat,
+                                module='^s3ql', append=True)
+        warnings.filterwarnings(action='ignore', category=cat, append=True)
+    warnings.filterwarnings(action='default', append=True)
+
 # We must not import s3ql.logging.logging as s3ql.logging,
 # otherwise future imports of s3ql.logging will incorrectly
 # use s3ql.logging.logging.

tests/conftest.py

@@ -52,14 +52,7 @@ def s3ql_cmd_argv(request):
                                                         os.path.join(basedir, 'bin', cmd) ]
 
 # Enable output checks
-pytest_plugins = ('pytest_checklogs')
-
-# Ignore DeprecationWarnings when running unit tests.  They are
-# unfortunately quite often a result of indirect imports via third party
-# modules, so we can't actually fix them.
-@pytest.fixture(autouse=True)
-def ignore_depreciation_warnings(reg_output):
-    reg_output(r'(Pending)?DeprecationWarning', count=0)
+pytest_plugins = ('pytest_checklogs',)
 
 @pytest.fixture()
 def pass_reg_output(request, reg_output):
@@ -86,11 +79,16 @@ def pytest_configure(config):
             os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))):
             sys.path = [os.path.join(basedir, 'src')] + sys.path
 
-    # When running from HG repo, enable all warnings
+    # When running from HG repo, enable warnings
     if os.path.exists(os.path.join(basedir, 'MANIFEST.in')):
         import warnings
         warnings.resetwarnings()
-        warnings.simplefilter('default')
+        for cat in (DeprecationWarning, PendingDeprecationWarning):
+            warnings.filterwarnings(action='default', category=cat,
+                                    module='s3ql', append=True)
+            warnings.filterwarnings(action='ignore', category=cat, append=True)
+        warnings.filterwarnings(action='default', append=True)
+        os.environ['S3QL_ENABLE_WARNINGS'] = '1'
 
     # Enable faulthandler
     faultlog_fd = os.open(os.path.join(basedir, 'test_crit.log'),

tests/pytest.ini

@@ -1,5 +1,6 @@
 [pytest]
-addopts = --verbose --assert=rewrite --exitfirst --tb=native
+# We handle warnings internally
+addopts = --verbose --assert=rewrite --exitfirst --tb=native  -p no:warnings
 python_files = t?_*.py
 log_cli_level = 100
 log_format = %(asctime)s.%(msecs)03d %(threadName)s %(name)s.%(funcName)s: %(message)s

tests/pytest_checklogs.py

@@ -96,9 +96,11 @@ def check_test_output(capfd, item):
         if count == 0 or count - cnt > 0:
             stderr = cp.sub('', stderr, count=count - cnt)
 
-    for pattern in ('exception', 'error', 'warning', 'fatal', 'traceback',
-                    'fault', 'crash(?:ed)?', 'abort(?:ed)', 'fishy'):
-        cp = re.compile(r'\b{}\b'.format(pattern), re.IGNORECASE | re.MULTILINE)
+    for pattern in (r'exception\b', r'error\b', r'warning\b',
+                    r'\bfatal\b', r'\btraceback\b',
+                    r'(:?seg)?fault\b', r'\bcrash(?:ed)?\b', r'\babort(?:ed)\b',
+                    r'\bfishy\b'):
+        cp = re.compile(pattern, re.IGNORECASE | re.MULTILINE)
         hit = cp.search(stderr)
         if hit:
             raise AssertionError('Suspicious output to stderr (matched "%s")' % hit.group(0))