Add gitlab and bitbucket (#38)

* update logic to add gitlab * add new env for gitlab token * exit if no token is found * add gitlab clone logic * update info message * update logging * add support for bitbucket * update changelog * update teams users for bitbucket * add gitlab namespace * update change log
gabrie30 · Aug 2, 2019 · be7354e · be7354e
2 parents ac5df2c + 891e5c6
commit be7354e
Show file tree

Hide file tree

Showing 10 changed files with 540 additions and 123 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 ### Fixed
 ### Security
 
+## [1.0.10] - 07/28/19
+### Added
+- gitlab support
+- bitbucket support
+### Changed
+- readme
+### Deprecated
+### Removed
+### Fixed
+- ghorg conf file env's being overwritten
+### Security
+
 ## [1.0.9] - 07/25/19
 ### Added
 - viper/cobra for more robust cli commands and flags

diff --git a/README.md b/README.md
@@ -12,6 +12,13 @@ ghorg allows you to quickly clone all of an orgs, or users repos into a single d
 
 > When running ghorg a second time, all local changes in your *_ghorg directory will be overwritten by whats on GitHub. If you are working out of this directory, make sure you rename it before running a second time otherwise all of you changes will be lost.
 
+## Supported Providers
+- github
+- gitlab
+- bitbucket (see bitbucket setup)
+
+> The terminology used in ghorg is that of GitHub, mainly orgs/repos. GitLab and BitBucket use different terminology. There is a handy chart thanks to GitLab that translates terminology [here](https://about.gitlab.com/images/blogimages/gitlab-terminology.png)
+
 ## Setup
 
 ### Homebrew
@@ -44,8 +51,9 @@ $ go install
 ## Use
 
 ```bash
-$ ghorg clone org-you-want-to-clone
-$ ghorg clone user-you-want-to-clone
+$ ghorg clone someorg
+$ ghorg clone someuser --clone-type=user --protocol=ssh --branch=develop
+$ ghorg clone gitlab-org --scm=gitlab --namespace=gitlab-org/security-products
 $ ghorg clone --help
 ```
 
@@ -55,26 +63,33 @@ $ ghorg clone --help
 
 Configuration can be set in two ways. The first is in `$HOME/ghorg/conf.yaml`. This file will be created from the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) and copied into `$HOME/ghorg/conf.yaml`. The second is setting flags via the cli, run `$ ghorg clone --help` for a list of flags
 
-## Default GitHub Token Used
+## Default GitHub/GitLab Token Used
 
 ```bash
 $ security find-internet-password -s github.com  | grep "acct" | awk -F\" '{ print $4 }'
+$ security find-internet-password -s gitlab.com  | grep "acct" | awk -F\" '{ print $4 }'
 ```
 
-> If running this does not return the correct key you will need to generate a token via GithHub and add it to your $HOME/ghorg/conf.yaml, or see Troubleshooting section below.
+> If running this does not return the correct key you will need to generate a token via GithHub/GitLab and add it to your $HOME/ghorg/conf.yaml, or see Troubleshooting section below.
 
 > To view all other default environment variables see sample-conf.yaml
 
 ## Auth through SSO
 
 - If org is behind SSO a normal token will not work. You will need to add SSO to the [Github token](https://help.github.com/articles/authorizing-a-personal-access-token-for-use-with-a-saml-single-sign-on-organization/)
 
+## Known issues
+- When cloning if you see something like `Username for 'https://gitlab.com': ` the command won't finish. I haven't been able to identify the reason for this occuring. The fix for this is to make sure your token is in the osxkeychain. See the troubleshooting section for how to set this up.
+
+## BitBucket Setup
+To configure with bitbucket you will need to create a new [app password](https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html) and update your `$HOME/ghorg/conf.yaml` to use those values or set the command line args.
+
 ## Troubleshooting
 
 - Make sure your `$ git --version` is >= 2.19.0
 - You may need to increase your ulimits if cloning a large org
 - If cloning via HTTPS make sure the osxkeychain has your github access token. This can be determined by running the `security` command above.
-    - If this command does not return anything either switch to cloning via ssh (update your `$HOME/ghorg/conf.yaml`) or set it up by following this [GitHub Documentation](https://help.github.com/en/articles/caching-your-github-password-in-git)
+    - If this command does not return anything either switch to cloning via ssh (update your `$HOME/ghorg/conf.yaml`) or set it up by following this [GitHub Documentation](https://help.github.com/en/articles/caching-your-github-password-in-git). For GitHub tokens you will need to set your token as your username and set nothing as the password when prompted. For GitLab you will need to set your token for both the username and password when prompted. This will correctly store your credentials in the keychain.
     - If your GitHub account is behind 2fa follow this [StackOverflow Post](https://stackoverflow.com/questions/31305945/git-clone-from-github-over-https-with-two-factor-authentication) or this [Github Documentation](https://github.blog/2013-09-03-two-factor-authentication/#how-does-it-work-for-command-line-git) as noted in comments be sure to use your token as your username and give a blank password.
 
 ### Updating brew tap

diff --git a/cmd/clone.go b/cmd/clone.go
@@ -1,34 +1,40 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"os/exec"
 	"strconv"
 	"strings"
 
-	"github.com/spf13/cobra"
-
 	"github.com/gabrie30/ghorg/colorlog"
-	"github.com/google/go-github/github"
-	"golang.org/x/oauth2"
+	"github.com/gabrie30/ghorg/configs"
+	"github.com/spf13/cobra"
 )
 
 var (
-	protocol string
-	path     string
-	branch   string
-	token    string
-	args     []string
+	protocol          string
+	path              string
+	branch            string
+	token             string
+	cloneType         string
+	scmType           string
+	bitbucketUsername string
+	namespace         string
+	args              []string
 )
 
 func init() {
 	rootCmd.AddCommand(cloneCmd)
 	cloneCmd.Flags().StringVar(&protocol, "protocol", "", "protocol to clone with, ssh or https, (defaults to https)")
 	cloneCmd.Flags().StringVarP(&path, "path", "p", "", "absolute path the ghorg_* directory will be created (defaults to Desktop)")
 	cloneCmd.Flags().StringVarP(&branch, "branch", "b", "", "branch left checked out for each repo cloned (defaults to master)")
-	cloneCmd.Flags().StringVarP(&token, "token", "t", "", "github token to clone with")
+	cloneCmd.Flags().StringVarP(&token, "token", "t", "", "scm token to clone with")
+	cloneCmd.Flags().StringVarP(&bitbucketUsername, "bitbucket_username", "", "", "when cloning with bitbucket this must be set or GHORG_BITBUKET_USERNAME in your $HOME/ghorg/conf.yaml")
+	cloneCmd.Flags().StringVarP(&scmType, "scm", "s", "github", "type of scm used, github or gitlab")
+	// TODO: make gitlab terminology make sense https://about.gitlab.com/2016/01/27/comparing-terms-gitlab-github-bitbucket/
+	cloneCmd.Flags().StringVarP(&cloneType, "clone-type", "c", "org", "clone target type, user or org, for gitlab groups use org flag")
+	cloneCmd.Flags().StringVarP(&namespace, "namespace", "n", "namespace", "gitlab only: limits clone targets to a specific namespace e.g. --namespace=gitlab-org/security-products")
 }
 
 var cloneCmd = &cobra.Command{
@@ -48,110 +54,91 @@ var cloneCmd = &cobra.Command{
 		}
 
 		if cmd.Flags().Changed("protocol") {
-			path := cmd.Flag("protocol").Value.String()
-			if path != "ssh" && path != "https" {
-				colorlog.PrintError("Protocol must be one of https or ssh")
-				os.Exit(1)
-			}
-			os.Setenv("GHORG_CLONE_PROTOCOL", path)
+			protocol := cmd.Flag("protocol").Value.String()
+			os.Setenv("GHORG_CLONE_PROTOCOL", protocol)
 		}
 
 		if cmd.Flags().Changed("branch") {
 			os.Setenv("GHORG_BRANCH", cmd.Flag("branch").Value.String())
 		}
 
+		if cmd.Flags().Changed("bitbucket_username") {
+			os.Setenv("GHORG_BITBUCKET_USERNAME", cmd.Flag("bitbucket_username").Value.String())
+		}
+
+		if cmd.Flags().Changed("namespace") {
+			os.Setenv("GHORG_GITLAB_DEFAULT_NAMESPACE", cmd.Flag("namespace").Value.String())
+		}
+
+		if cmd.Flags().Changed("clone-type") {
+			cloneType := strings.ToLower(cmd.Flag("clone-type").Value.String())
+			os.Setenv("GHORG_CLONE_TYPE", cloneType)
+		}
+
+		if cmd.Flags().Changed("scm") {
+			scmType := strings.ToLower(cmd.Flag("scm").Value.String())
+			os.Setenv("GHORG_SCM_TYPE", scmType)
+		}
+
+		configs.GetOrSetToken()
+
 		if cmd.Flags().Changed("token") {
-			os.Setenv("GHORG_GITHUB_TOKEN", cmd.Flag("token").Value.String())
+			if os.Getenv("GHORG_SCM_TYPE") == "github" {
+				os.Setenv("GHORG_GITHUB_TOKEN", cmd.Flag("token").Value.String())
+			} else if os.Getenv("GHORG_SCM_TYPE") == "gitlab" {
+				os.Setenv("GHORG_GITLAB_TOKEN", cmd.Flag("token").Value.String())
+			} else if os.Getenv("GHORG_SCM_TYPE") == "bitbucket" {
+				os.Setenv("GHORG_BITBUCKET_APP_PASSWORD", cmd.Flag("token").Value.String())
+			}
 		}
 
+		configs.VerifyTokenSet()
+		configs.VerifyConfigsSetCorrectly()
+
 		args = argz
 
-		CloneAllReposByOrg()
+		CloneAllRepos()
 	},
 }
 
 // TODO: Figure out how to use go channels for this
 func getAllOrgCloneUrls() ([]string, error) {
 	asciiTime()
-	ctx := context.Background()
-	ts := oauth2.StaticTokenSource(
-		&oauth2.Token{AccessToken: os.Getenv("GHORG_GITHUB_TOKEN")},
-	)
-	tc := oauth2.NewClient(ctx, ts)
-	client := github.NewClient(tc)
-
-	opt := &github.RepositoryListByOrgOptions{
-		Type:        "all",
-		ListOptions: github.ListOptions{PerPage: 100, Page: 0},
-	}
-
-	// get all pages of results
-	var allRepos []*github.Repository
-	for {
-		repos, resp, err := client.Repositories.ListByOrg(context.Background(), args[0], opt)
-
-		if err != nil {
-			return nil, err
-		}
-		allRepos = append(allRepos, repos...)
-		if resp.NextPage == 0 {
-			break
-		}
-		opt.Page = resp.NextPage
-	}
-	cloneUrls := []string{}
-
-	for _, repo := range allRepos {
-		if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
-			cloneUrls = append(cloneUrls, *repo.CloneURL)
-		} else {
-			cloneUrls = append(cloneUrls, *repo.SSHURL)
-		}
+	var urls []string
+	var err error
+	switch os.Getenv("GHORG_SCM_TYPE") {
+	case "github":
+		urls, err = getGitHubOrgCloneUrls()
+	case "gitlab":
+		urls, err = getGitLabOrgCloneUrls()
+	case "bitbucket":
+		urls, err = getBitBucketOrgCloneUrls()
+	default:
+		colorlog.PrintError("GHORG_SCM_TYPE not set or unsupported, also make sure its all lowercase")
+		os.Exit(1)
 	}
 
-	return cloneUrls, nil
+	return urls, err
 }
 
-// TODO: refactor with getAllOrgCloneUrls
+// TODO: Figure out how to use go channels for this
 func getAllUserCloneUrls() ([]string, error) {
-	ctx := context.Background()
-
-	ts := oauth2.StaticTokenSource(
-		&oauth2.Token{AccessToken: os.Getenv("GHORG_GITHUB_TOKEN")},
-	)
-	tc := oauth2.NewClient(ctx, ts)
-	client := github.NewClient(tc)
-
-	opt := &github.RepositoryListOptions{
-		Type:        "all",
-		ListOptions: github.ListOptions{PerPage: 100, Page: 0},
-	}
-
-	// get all pages of results
-	var allRepos []*github.Repository
-	for {
-		repos, resp, err := client.Repositories.List(context.Background(), args[0], opt)
-
-		if err != nil {
-			return nil, err
-		}
-		allRepos = append(allRepos, repos...)
-		if resp.NextPage == 0 {
-			break
-		}
-		opt.Page = resp.NextPage
-	}
-	cloneUrls := []string{}
-
-	for _, repo := range allRepos {
-		if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
-			cloneUrls = append(cloneUrls, *repo.CloneURL)
-		} else {
-			cloneUrls = append(cloneUrls, *repo.SSHURL)
-		}
+	asciiTime()
+	var urls []string
+	var err error
+	switch os.Getenv("GHORG_SCM_TYPE") {
+	case "github":
+		urls, err = getGitHubUserCloneUrls()
+	case "gitlab":
+		urls, err = getGitLabUserCloneUrls()
+	case "bitbucket":
+		urls, err = getBitBucketUserCloneUrls()
+	default:
+		colorlog.PrintError("GHORG_SCM_TYPE not set or unsupported, also make sure its all lowercase")
+		os.Exit(1)
 	}
 
-	return cloneUrls, nil
+	return urls, err
 }
 
 func createDirIfNotExist() {
@@ -200,12 +187,10 @@ func printRemainingMessages(infoMessages []error, errors []error) {
 	}
 }
 
-// CloneAllReposByOrg clones all repos for a given org
-func CloneAllReposByOrg() {
+// CloneAllRepos clones all repos
+func CloneAllRepos() {
 	resc, errc, infoc := make(chan string), make(chan error), make(chan error)
 
-	createDirIfNotExist()
-
 	if os.Getenv("GHORG_BRANCH") != "master" {
 		colorlog.PrintSubtleInfo("***********************************************************")
 		colorlog.PrintSubtleInfo("* Ghorg will be running on branch: " + os.Getenv("GHORG_BRANCH"))
@@ -214,22 +199,33 @@ func CloneAllReposByOrg() {
 		fmt.Println()
 	}
 
-	cloneTargets, err := getAllOrgCloneUrls()
+	var cloneTargets []string
+	var err error
 
-	if err != nil {
-		colorlog.PrintSubtleInfo("Change of Plans! Did not find GitHub Org " + args[0] + " -- Looking instead for a GitHub User: " + args[0])
-		fmt.Println()
+	if os.Getenv("GHORG_CLONE_TYPE") == "org" {
+		cloneTargets, err = getAllOrgCloneUrls()
+	} else if os.Getenv("GHORG_CLONE_TYPE") == "user" {
 		cloneTargets, err = getAllUserCloneUrls()
+	} else {
+		colorlog.PrintError("GHORG_CLONE_TYPE not set or unsupported")
+		os.Exit(1)
+	}
+
+	if len(cloneTargets) == 0 {
+		colorlog.PrintInfo("No repos found for " + os.Getenv("GHORG_SCM_TYPE") + " " + os.Getenv("GHORG_CLONE_TYPE") + ": " + args[0] + ", check spelling and verify clone_type (user/org) is set correctly e.g. -c=user")
+		os.Exit(0)
 	}
 
 	if err != nil {
-		colorlog.PrintError(err)
+		colorlog.PrintSubtleInfo("Did not find " + os.Getenv("GHORG_SCM_TYPE") + " " + os.Getenv("GHORG_CLONE_TYPE") + ": " + args[0] + ", check spelling and try again.")
 		os.Exit(1)
 	} else {
 		colorlog.PrintInfo(strconv.Itoa(len(cloneTargets)) + " repos found in " + args[0])
 		fmt.Println()
 	}
 
+	createDirIfNotExist()
+
 	for _, target := range cloneTargets {
 		appName := getAppNameFromURL(target)
 
@@ -297,7 +293,6 @@ func CloneAllReposByOrg() {
 	}
 
 	printRemainingMessages(infoMessages, errors)
-
 	colorlog.PrintSuccess(fmt.Sprintf("Finished! %s%s_ghorg", os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), args[0]))
 }