diff --git a/.hound.yml b/.hound.yml new file mode 100644 index 00000000..d5b85468 --- /dev/null +++ b/.hound.yml @@ -0,0 +1,3 @@ +flake8: + enabled: true + config_file: .flake8 diff --git a/GNUmakefile b/GNUmakefile index ad9a7135..2dac2702 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -106,7 +106,7 @@ test3: TAR_SCM_TESTMODE=1 PYTHONPATH=. python3 tests/test.py 2>&1 | tee ./test3.log .PHONY: pylint -pylint: pylint2 +pylint: pylint2 pylinttest2 .PHONY: pylint3 pylint3: @@ -121,11 +121,19 @@ pylint3: pylint2: @if [ "x$(PYLINT2)" != "x" ]; then \ $(PYLINT2) --rcfile=./.pylintrc $(PYLINT_READY_MODULES); \ + else \ + echo "PYLINT2 not set - Skipping tests"; \ + fi + +.PHONY: pylinttest2 +pylinttest2: + @if [ "x$(PYLINT2)" != "x" ]; then \ PYTHONPATH=tests $(PYLINT2) --rcfile=./.pylinttestsrc $(PYLINT_READY_TEST_MODULES); \ else \ echo "PYLINT2 not set - Skipping tests"; \ fi + cover: PYTHONPATH=. coverage2 run tests/test.py 2>&1 | tee ./cover.log coverage2 html --include=./TarSCM/* diff --git a/TarSCM/archive.py b/TarSCM/archive.py index b971ea45..fc721d90 100644 --- a/TarSCM/archive.py +++ b/TarSCM/archive.py @@ -63,11 +63,12 @@ def create_archive(self, scm_object, **kwargs): # detect reproducible support params = ['cpio', '--create', '--format=newc'] - if os.system("cpio --create --format=newc" + - "--reproducible /dev/null") == 0: + chkcmd = "cpio --create --format=newc --reproducible " + chkcmd += "/dev/null 2>&1" + if os.system(chkcmd) == 0: params.append('--reproducible') - proc = subprocess.Popen( + proc = subprocess.Popen( params, shell = False, stdin = subprocess.PIPE, diff --git a/TarSCM/changes.py b/TarSCM/changes.py index 704584f9..8a1a9816 100644 --- a/TarSCM/changes.py +++ b/TarSCM/changes.py @@ -181,8 +181,9 @@ def write_changes(self, changes_filename, changes, version, author): logging.debug("Writing changes file %s", changes_filename) tmp_fp = tempfile.NamedTemporaryFile(delete=False) - os.chmod(tmp_fp.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | - stat.S_IROTH) + mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + os.chmod(tmp_fp.name, mode) + tmp_fp.write('-' * 67 + '\n') tmp_fp.write("%s - %s\n" % ( datetime.datetime.utcnow().strftime('%a %b %d %H:%M:%S UTC %Y'), diff --git a/TarSCM/scm/base.py b/TarSCM/scm/base.py index 7cd11466..593b0731 100644 --- a/TarSCM/scm/base.py +++ b/TarSCM/scm/base.py @@ -181,10 +181,11 @@ def prepare_clone_dir(self): # special case when using osc and creating an obscpio, use # current work directory to allow the developer to work inside # of the git repo and fetch local changes - if sys.argv[0].endswith("snapcraft") or \ - (self.args.use_obs_scm and - os.getenv('OSC_VERSION') and - os.path.isdir('.git')): + is_snap = sys.argv[0].endswith("snapcraft") + is_obs_scm = self.args.use_obs_scm + in_osc = bool(os.getenv('OSC_VERSION')) + in_git = os.path.isdir('.git') + if is_snap or (is_obs_scm and in_osc and in_git): self.repodir = os.getcwd() # construct repodir (the parent directory of the checkout) @@ -265,10 +266,11 @@ def prep_tree_for_archive(self, subdir, outdir, dstname): raise Exception("%s: No such file or directory" % src) self.arch_dir = dst = os.path.join(outdir, dstname) - if os.path.exists(dst) and \ - (os.path.samefile(src, dst) or - os.path.samefile(os.path.dirname(src), dst)): - return + if os.path.exists(dst): + same = os.path.samefile(src, dst) or \ + os.path.samefile(os.path.dirname(src), dst) + if same: + return r_path = os.path.realpath(src) c_dir = os.path.realpath(self.clone_dir) diff --git a/TarSCM/scm/bzr.py b/TarSCM/scm/bzr.py index f3f11cdf..966e004c 100644 --- a/TarSCM/scm/bzr.py +++ b/TarSCM/scm/bzr.py @@ -11,9 +11,7 @@ class Bzr(Scm): def _get_scm_cmd(self): """Compose a BZR-specific command line using http proxies.""" # Bazaar honors the http[s]_proxy variables, no action needed - scmcmd = ['bzr'] - - return scmcmd + return [self.scm] def fetch_upstream_scm(self): """SCM specific version of fetch_uptream for bzr.""" @@ -46,8 +44,8 @@ def detect_version(self, args): if versionformat is None: versionformat = '%r' - version = self.helpers.safe_run(self._get_scm_cmd() + - ['revno'], self.clone_dir)[1] + version = self.helpers.safe_run(self._get_scm_cmd() + ['revno'], + self.clone_dir)[1] return re.sub('%r', version.strip(), versionformat) def get_timestamp(self): diff --git a/TarSCM/scm/git.py b/TarSCM/scm/git.py index 661c4501..402ab12f 100644 --- a/TarSCM/scm/git.py +++ b/TarSCM/scm/git.py @@ -18,11 +18,9 @@ def _get_scm_cmd(self): # is needed here scmcmd = ['git'] if self.httpproxy: - scmcmd += ['-c', 'http.proxy=' + - self.httpproxy] + scmcmd += ['-c', 'http.proxy=' + self.httpproxy] if self.httpsproxy: - scmcmd += ['-c', 'https.proxy=' + - self.httpsproxy] + scmcmd += ['-c', 'https.proxy=' + self.httpsproxy] return scmcmd def switch_revision(self): @@ -48,8 +46,7 @@ def switch_revision(self): if os.getenv('OSC_VERSION') and \ len(os.listdir(self.clone_dir)) > 1: stash_text = self.helpers.safe_run( - self._get_scm_cmd() + - ['stash'], + self._get_scm_cmd() + ['stash'], cwd=self.clone_dir)[1] text = self.helpers.safe_run( self._get_scm_cmd() + ['reset', '--hard', rev], @@ -58,8 +55,7 @@ def switch_revision(self): if stash_text != "No local changes to save\n": logging.debug("[switch_revision] GIT STASHING") text += self.helpers.safe_run( - self._get_scm_cmd() + - ['stash', 'pop'], + self._get_scm_cmd() + ['stash', 'pop'], cwd=self.clone_dir)[1] else: text = self.helpers.safe_run( @@ -108,19 +104,14 @@ def fetch_specific_revision(self): def fetch_submodules(self): """Recursively initialize git submodules.""" - if ( - 'submodules' in self.args.__dict__ and - self.args.__dict__['submodules'] == 'enable' - ): + argsd = self.args.__dict__ + if ('submodules' in argsd and argsd['submodules'] == 'enable'): self.helpers.safe_run( self._get_scm_cmd() + ['submodule', 'update', '--init', '--recursive'], cwd=self.clone_dir ) - elif ( - 'submodules' in self.args.__dict__ and - self.args.__dict__['submodules'] == 'master' - ): + elif ('submodules' in argsd and argsd['submodules'] == 'master'): self.helpers.safe_run( self._get_scm_cmd() + ['submodule', 'update', '--init', '--recursive', '--remote'], @@ -222,17 +213,16 @@ def _detect_version_tag_offset(self, parent_tag, versionformat): sys.exit("\033[31m@TAG_OFFSET@ cannot be expanded, " "as no parent tag was discovered.\033[0m") - rcode, output = self.helpers.run_cmd( - self._get_scm_cmd() + ['rev-list', '--count', parent_tag + - '..HEAD'], - self.clone_dir - ) + cmd = self._get_scm_cmd() + cmd.extend(['rev-list', '--count', parent_tag + '..HEAD']) + rcode, out = self.helpers.run_cmd(cmd, self.clone_dir) if rcode: - sys.exit("\033[31m@TAG_OFFSET@ can not be expanded: " + - output + "\033[0m") + msg = "\033[31m@TAG_OFFSET@ can not be expanded: %s\033[0m" + msg.format(out) + sys.exit(msg) - tag_offset = output.strip() + tag_offset = out.strip() versionformat = re.sub('@TAG_OFFSET@', tag_offset, versionformat) return versionformat diff --git a/TarSCM/scm/hg.py b/TarSCM/scm/hg.py index 30aedeef..b8cec630 100644 --- a/TarSCM/scm/hg.py +++ b/TarSCM/scm/hg.py @@ -17,28 +17,30 @@ def _get_scm_cmd(self): # Mercurial requires declaring proxies via a --config parameter scmcmd = ['hg'] if self.httpproxy: - logging.debug("using " + self.hgtmpdir) - f = open(self.hgtmpdir + "/tempsettings.rc", "wb") - f.write('[http_proxy]\n') - - regexp_proxy = re.match('http://(.*):(.*)', - self.httpproxy, - re.M | re.I) - - if regexp_proxy.group(1) is not None: - print('using proxy host: ' + regexp_proxy.group(1)) - f.write('host=' + regexp_proxy.group(1)) - if regexp_proxy.group(2) is not None: - print('using proxy port: ' + regexp_proxy.group(2)) - f.write('port=' + regexp_proxy.group(2)) - if self.noproxy is not None: - print('using proxy exceptions: ' + - self.noproxy) - f.write('no=' + self.noproxy) - f.close() - - # we just point Mercurial to where the config file is - os.environ['HGRCPATH'] = self.hgtmpdir + logging.debug("using tempdir: %s", self.hgtmpdir) + cfg = open(self.hgtmpdir + "/tempsettings.rc", "wb") + cfg.write('[http_proxy]\n') + + regexp_proxy = re.match('http://(.*):(.*)', + self.httpproxy, + re.M | re.I) + + proxy_host = regexp_proxy.group(1) + proxy_port = regexp_proxy.group(2) + + if proxy_host is not None: + logging.debug('using proxy host: %s', proxy_host) + cfg.write('host=' + proxy_host) + if proxy_port is not None: + logging.debug('using proxy port: %s', proxy_port) + cfg.write('port=' + proxy_port) + if self.noproxy is not None: + logging.debug('using proxy exceptions: %s', self.noproxy) + cfg.write('no=' + self.noproxy) + cfg.close() + + # we just point Mercurial to where the config file is + os.environ['HGRCPATH'] = self.hgtmpdir return scmcmd @@ -47,8 +49,8 @@ def switch_revision(self): if self.revision is None: self.revision = 'tip' - rcode, _ = self.helpers.run_cmd(self._get_scm_cmd() + - ['update', self.revision], + cmd = self._get_scm_cmd() + ['update', self.revision] + rcode, _ = self.helpers.run_cmd(cmd, cwd=self.clone_dir, interactive=sys.stdout.isatty()) if rcode: @@ -66,8 +68,8 @@ def fetch_upstream_scm(self): def update_cache(self): """Update sources via hg.""" try: - self.helpers.safe_run(self._get_scm_cmd() + - ['pull'], cwd=self.clone_dir, + self.helpers.safe_run(self._get_scm_cmd() + ['pull'], + cwd=self.clone_dir, interactive=sys.stdout.isatty()) except SystemExit as exc: # Contrary to the docs, hg pull returns exit code 1 when @@ -84,8 +86,8 @@ def detect_version(self, args): if versionformat is None: versionformat = '{rev}' - version = self.helpers.safe_run(self._get_scm_cmd() + - ['id', '-n'], self.clone_dir)[1] + cmd = self._get_scm_cmd() + ['id', '-n'] + version = self.helpers.safe_run(cmd, self.clone_dir)[1] # Mercurial internally stores commit dates in its changelog # context objects as (epoch_secs, tz_delta_to_utc) tuples (see @@ -113,17 +115,16 @@ def detect_version(self, args): # 'sub(...)' which is only available since 2.4 (first introduced # in openSUSE 12.3). - version = self.helpers.safe_run( - self._get_scm_cmd() + - [ - 'log', - '-l1', - "-r%s" % version.strip(), - '--template', - versionformat - ], - self.clone_dir - )[1] + cmd = self._get_scm_cmd() + cmd.extend([ + 'log', + '-l1', + "-r%s" % version.strip(), + '--template', + versionformat + ]) + + version = self.helpers.safe_run(cmd, self.clone_dir)[1] return self.version_iso_cleanup(version) def get_timestamp(self): @@ -134,10 +135,10 @@ def get_timestamp(self): def cleanup(self): try: - shutil.rmtree(self.hgtmpdir, ignore_errors=True) + shutil.rmtree(self.hgtmpdir, ignore_errors=True) except: - logging.debug("error on cleanup:", sys.exc_info()[0]) - raise + logging.debug("error on cleanup: %s", sys.exc_info()[0]) + raise def check_url(self): """check if url is a remote url""" diff --git a/TarSCM/scm/svn.py b/TarSCM/scm/svn.py index 59b24bc3..4488b19f 100644 --- a/TarSCM/scm/svn.py +++ b/TarSCM/scm/svn.py @@ -23,50 +23,48 @@ def _get_scm_cmd(self): # config directory that will be added via '--config-dir' scmcmd = ['svn'] if self.httpproxy: - logging.debug("using " + self.svntmpdir) - f = open(self.svntmpdir + "/servers", "wb") - f.write('[global]\n') - - regexp_proxy = re.match('http://(.*):(.*)', - self.httpproxy, - re.M | re.I) - - if regexp_proxy.group(1) is not None: - logging.debug('using proxy host: ' + - regexp_proxy.group(1)) - f.write('http-proxy-host=' + - regexp_proxy.group(1) + '\n') - - if regexp_proxy.group(2) is not None: - logging.debug('using proxy port: ' + - regexp_proxy.group(2)) - f.write('http-proxy-port=' + - regexp_proxy.group(2) + '\n') - - if self.noproxy is not None: - logging.debug('using proxy exceptions: ' + - self.noproxy) - no_proxy_domains = [] - no_proxy_domains.append(tuple(self.noproxy.split(","))) - no_proxy_string = "" - - # for some odd reason subversion expects the domains - # to have an asterisk - for i in range(len(no_proxy_domains[0])): - tmpstr = str(no_proxy_domains[0][i]).strip() - if tmpstr.startswith('.'): - no_proxy_string += '*' + tmpstr - else: - no_proxy_string += tmpstr - - if i < len(no_proxy_domains[0]) - 1: - no_proxy_string += ',' - - no_proxy_string += '\n' - logging.debug('no_proxy string = ' + no_proxy_string) - f.write('http-proxy-exceptions=' + no_proxy_string) - f.close() - scmcmd += ['--config-dir', self.svntmpdir] + logging.debug("using %s", self.svntmpdir) + cfg = open(self.svntmpdir + "/servers", "wb") + cfg.write('[global]\n') + + re_proxy = re.match('http://(.*):(.*)', + self.httpproxy, + re.M | re.I) + + proxy_host = re_proxy.group(1) + proxy_port = re_proxy.group(2) + + if proxy_host is not None: + logging.debug('using proxy host: %s', proxy_host) + cfg.write('http-proxy-host=' + proxy_host + '\n') + + if proxy_port is not None: + logging.debug('using proxy port: %s', proxy_port) + cfg.write('http-proxy-port=' + proxy_port + '\n') + + if self.noproxy is not None: + logging.debug('using proxy exceptions: %s', self.noproxy) + no_proxy_domains = [] + no_proxy_domains.append(tuple(self.noproxy.split(","))) + no_proxy_string = "" + + # for some odd reason subversion expects the domains + # to have an asterisk + for i in range(len(no_proxy_domains[0])): + tmpstr = str(no_proxy_domains[0][i]).strip() + if tmpstr.startswith('.'): + no_proxy_string += '*' + tmpstr + else: + no_proxy_string += tmpstr + + if i < len(no_proxy_domains[0]) - 1: + no_proxy_string += ',' + + no_proxy_string += '\n' + logging.debug('no_proxy string = %s', no_proxy_string) + cfg.write('http-proxy-exceptions=' + no_proxy_string) + cfg.close() + scmcmd += ['--config-dir', self.svntmpdir] return scmcmd @@ -191,10 +189,10 @@ def _get_rev(self, clone_dir, num_commits): def cleanup(self): try: - shutil.rmtree(self.svntmpdir, ignore_errors=True) + shutil.rmtree(self.svntmpdir, ignore_errors=True) except: - logging.debug("error on cleanup:", sys.exc_info()[0]) - raise + logging.debug("error on cleanup: %s", sys.exc_info()[0]) + raise def check_url(self): """check if url is a remote url""" diff --git a/TarSCM/tasks.py b/TarSCM/tasks.py index 4e663125..2a62af6e 100644 --- a/TarSCM/tasks.py +++ b/TarSCM/tasks.py @@ -171,10 +171,10 @@ def process_single_task(self, args): # if exception occurs self.scm_object = scm_object = scm_class(args, self) - # TODO: find a way to mock this function in tests to a stub - if not bool(os.getenv('TAR_SCM_TESTMODE')): - if not scm_object.check_url(): - sys.exit("--url does not match remote repository") + # TODO: find a way to mock this function in tests to a stub # noqa: E501 # pylint: disable=fixme + tmode = bool(os.getenv('TAR_SCM_TESTMODE')) + if not tmode and not scm_object.check_url(): + sys.exit("--url does not match remote repository") try: scm_object.check_scm() @@ -221,24 +221,30 @@ def process_single_task(self, args): ) if detected_changes: - changesauthor = self.changes.get_changesauthor(args) + self._process_changes(args, + version, + changesversion, + detected_changes) - logging.debug("AUTHOR: %s", changesauthor) + scm_object.finalize() - if not version: - args.version = "_auto_" - changesversion = self.get_version() + def _process_changes(self, args, ver, changesversion, detected_changes): + changesauthor = self.changes.get_changesauthor(args) - for filename in glob.glob('*.changes'): - new_changes_file = os.path.join(args.outdir, filename) - shutil.copy(filename, new_changes_file) - self.changes.write_changes(new_changes_file, - detected_changes['lines'], - changesversion, changesauthor) - self.changes.write_changes_revision(args.url, args.outdir, - detected_changes['revision']) + logging.debug("AUTHOR: %s", changesauthor) - scm_object.finalize() + if not ver: + args.version = "_auto_" + changesversion = self.get_version() + + for filename in glob.glob('*.changes'): + new_changes_file = os.path.join(args.outdir, filename) + shutil.copy(filename, new_changes_file) + self.changes.write_changes(new_changes_file, + detected_changes['lines'], + changesversion, changesauthor) + self.changes.write_changes_revision(args.url, args.outdir, + detected_changes['revision']) def get_version(self): '''