fix: Update plugin code to support no connection

Makefile

@@ -15,4 +15,8 @@ fmt:
 
 .PHONY: build
 build:
-	go build -o bin/cq-source-sharepoint -v
+	go build -o bin/cq-source-sharepoint -v
+
+.PHONY: package
+package:
+	go run main.go package --docs-dir docs -m @CHANGELOG.md v2.0.0 .

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/apache/arrow/go/v14 v14.0.0-20231030205031-cb11e44d878f
 	github.com/brianvoe/gofakeit/v6 v6.24.0
-	github.com/cloudquery/plugin-sdk/v4 v4.17.0
+	github.com/cloudquery/plugin-sdk/v4 v4.17.1
 	github.com/google/uuid v1.4.0
 	github.com/koltyakov/gosip v0.0.0-20231003001958-007c8072d71c
 	github.com/koltyakov/gosip-sandbox v0.0.0-20230410140555-1211f873b91c

go.sum

@@ -79,8 +79,8 @@ github.com/cloudquery/plugin-pb-go v1.13.1 h1:UR07rJgiExsY6TSDNvSHyaYsZl/QSIK62I
 github.com/cloudquery/plugin-pb-go v1.13.1/go.mod h1:dpnHh8INCc+TYrOCHFKmnEFjerTrpIHCJ3u9NRGB2h8=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
 github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
-github.com/cloudquery/plugin-sdk/v4 v4.17.0 h1:R+6M4Unf+zkhwW6nOvhqRLeNUCF0Cos+CvkieFRNM2A=
-github.com/cloudquery/plugin-sdk/v4 v4.17.0/go.mod h1:vGiAHFS2sOodpk1NC8rwuYjRp53oFuEk2mEq4MXgAJc=
+github.com/cloudquery/plugin-sdk/v4 v4.17.1 h1:BQkDpWThRfqq5jKld9r7FAwfoXHV3+kMqaWTO+Wr//M=
+github.com/cloudquery/plugin-sdk/v4 v4.17.1/go.mod h1:vGiAHFS2sOodpk1NC8rwuYjRp53oFuEk2mEq4MXgAJc=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI=

resources/plugin/client.go

@@ -4,126 +4,99 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/cloudquery/plugin-sdk/v4/message"
 	"github.com/cloudquery/plugin-sdk/v4/plugin"
 	"github.com/cloudquery/plugin-sdk/v4/scheduler"
 	"github.com/cloudquery/plugin-sdk/v4/schema"
-	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/koltyakov/cq-source-sharepoint/resources/auth"
-	"github.com/koltyakov/cq-source-sharepoint/resources/services/ct"
-	"github.com/koltyakov/cq-source-sharepoint/resources/services/lists"
-	"github.com/koltyakov/cq-source-sharepoint/resources/services/mmd"
-	"github.com/koltyakov/cq-source-sharepoint/resources/services/profiles"
-	"github.com/koltyakov/cq-source-sharepoint/resources/services/search"
 	"github.com/rs/zerolog"
 )
 
 type Client struct {
-	lists        *lists.Lists
-	mmd          *mmd.MMD
-	profiles     *profiles.Profiles
-	search       *search.Search
-	contentTypes *ct.ContentTypesRollup
+	logger    zerolog.Logger
+	spec      Spec
+	tables    schema.Tables
+	scheduler *scheduler.Scheduler
+
+	options plugin.NewClientOptions
+
+	plugin.UnimplementedDestination
 }
 
-func NewClient(ctx context.Context, logger zerolog.Logger, cnfg []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
-	spec, err := getSpec(cnfg)
-	if err != nil {
-		return nil, fmt.Errorf("failed to unmarshal spec: %w", err)
+func (*Client) ID() string {
+	return Name
+}
+func (c *Client) Sync(ctx context.Context, options plugin.SyncOptions, res chan<- message.SyncMessage) error {
+	if c.options.NoConnection {
+		return fmt.Errorf("no connection")
 	}
 
-	sp, err := auth.GetSP(spec.Auth)
+	tt, err := c.Tables(ctx, plugin.TableOptions{
+		Tables:              options.Tables,
+		SkipTables:          options.SkipTables,
+		SkipDependentTables: options.SkipDependentTables,
+	})
+
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	if _, err := sp.Web().Select("Title").Get(); err != nil {
-		return nil, fmt.Errorf("failed to connect to SharePoint: %w", err)
-	}
+	return c.scheduler.Sync(ctx, c, tt, res, scheduler.WithSyncDeterministicCQID(options.DeterministicCQID))
+}
 
-	client := &Client{
-		lists:        lists.NewLists(sp, logger),
-		mmd:          mmd.NewMMD(sp, logger),
-		profiles:     profiles.NewProfiles(sp, logger),
-		search:       search.NewSearch(sp, logger),
-		contentTypes: ct.NewContentTypesRollup(sp, logger),
+func (c *Client) Tables(_ context.Context, options plugin.TableOptions) (schema.Tables, error) {
+	if c.options.NoConnection {
+		return schema.Tables{}, nil
 	}
 
-	tables, err := client.getTables(spec)
+	tt, err := c.tables.FilterDfs(options.Tables, options.SkipTables, options.SkipDependentTables)
 	if err != nil {
-		return nil, fmt.Errorf("failed to retrieve tables: %w", err)
-	}
-
-	if opts.NoConnection {
-		return &Plugin{
-			logger: logger,
-			tables: tables,
-		}, nil
+		return nil, err
 	}
 
-	return &Plugin{
-		logger:    logger,
-		spec:      *spec,
-		tables:    tables,
-		scheduler: scheduler.NewScheduler(scheduler.WithLogger(logger)),
-		client:    client,
-	}, nil
+	return tt, nil
 }
 
-func (c *Client) getTables(config *Spec) (schema.Tables, error) {
-	tables := schema.Tables{}
-
-	// Tables from lists config
-	for listURI, listSpec := range config.Lists {
-		table, err := c.lists.GetDestTable(listURI, listSpec)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get list '%s': %w", listURI, err)
-		}
-		tables = append(tables, table)
-	}
+func (*Client) Close(context.Context) error {
+	// ToDo: Add your client cleanup here
+	return nil
+}
 
-	// Tables from mmd config
-	for terSetID, mmdSpec := range config.MMD {
-		table, err := c.mmd.GetDestTable(terSetID, mmdSpec)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get term set '%s': %w", terSetID, err)
-		}
-		tables = append(tables, table)
-	}
+func NewClient(_ context.Context, logger zerolog.Logger, cnfg []byte, opts plugin.NewClientOptions) (plugin.Client, error) {
+	logger = logger.With().Str("plugin", "sharepoint").Logger()
 
-	// Tables from profiles config
-	if config.Profiles.Enabled {
-		table, err := c.profiles.GetDestTable(config.Profiles)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get profiles: %w", err)
-		}
-		tables = append(tables, table)
+	if opts.NoConnection {
+		// no spec could be present
+		return &Client{
+			logger:  logger,
+			options: opts,
+		}, nil
 	}
 
-	// Tables from search config
-	for searchName, searchSpec := range config.Search {
-		table, err := c.search.GetDestTable(searchName, searchSpec)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get search '%s': %w", searchName, err)
-		}
-		tables = append(tables, table)
+	spec, err := getSpec(cnfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal spec: %w", err)
 	}
 
-	// Tables from content types config
-	for ctName, ctSpec := range config.ContentTypes {
-		table, err := c.contentTypes.GetDestTable(ctName, ctSpec)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get content type '%s': %w", ctName, err)
-		}
-		tables = append(tables, table)
+	sp, err := auth.GetSP(spec.Auth)
+	if err != nil {
+		return nil, err
 	}
 
-	if err := transformers.TransformTables(tables); err != nil {
-		return nil, err
+	if _, err := sp.Web().Select("Title").Get(); err != nil {
+		return nil, fmt.Errorf("failed to connect to SharePoint: %w", err)
 	}
 
-	for _, table := range tables {
-		schema.AddCqIDs(table)
+	tables, err := spec.getTables(sp, logger)
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve tables: %w", err)
 	}
 
-	return tables, nil
+	return &Client{
+		logger:    logger,
+		spec:      *spec,
+		tables:    tables,
+		scheduler: scheduler.NewScheduler(scheduler.WithLogger(logger)),
+		options:   opts,
+	}, nil
 }

resources/plugin/plugin.go

@@ -1,13 +1,7 @@
 package plugin
 
 import (
-	"context"
-
-	"github.com/cloudquery/plugin-sdk/v4/message"
 	"github.com/cloudquery/plugin-sdk/v4/plugin"
-	"github.com/cloudquery/plugin-sdk/v4/scheduler"
-	"github.com/cloudquery/plugin-sdk/v4/schema"
-	"github.com/rs/zerolog"
 )
 
 var (
@@ -17,49 +11,6 @@ var (
 	Version = "development"
 )
 
-type Plugin struct {
-	logger    zerolog.Logger
-	spec      Spec
-	tables    schema.Tables
-	scheduler *scheduler.Scheduler
-
-	client *Client
-
-	plugin.UnimplementedDestination
-}
-
 func NewPlugin() *plugin.Plugin {
 	return plugin.NewPlugin(Name, Version, NewClient, plugin.WithKind(Kind), plugin.WithTeam(Team))
 }
-
-func (*Plugin) ID() string {
-	return Name
-}
-
-func (p *Plugin) Sync(ctx context.Context, options plugin.SyncOptions, res chan<- message.SyncMessage) error {
-	tt, err := p.Tables(ctx, plugin.TableOptions{
-		Tables:              options.Tables,
-		SkipTables:          options.SkipTables,
-		SkipDependentTables: options.SkipDependentTables,
-	})
-
-	if err != nil {
-		return err
-	}
-
-	return p.scheduler.Sync(ctx, p, tt, res, scheduler.WithSyncDeterministicCQID(options.DeterministicCQID))
-}
-
-func (p *Plugin) Tables(ctx context.Context, options plugin.TableOptions) (schema.Tables, error) {
-	tt, err := p.tables.FilterDfs(options.Tables, options.SkipTables, options.SkipDependentTables)
-	if err != nil {
-		return nil, err
-	}
-
-	return tt, nil
-}
-
-func (*Plugin) Close(ctx context.Context) error {
-	// ToDo: Add your client cleanup here
-	return nil
-}