From 1d1dfa1af43819d645c197ed20dded45f988776e Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Tue, 31 Oct 2023 18:47:58 +0200 Subject: [PATCH 1/2] fix for package cmd --- resources/plugin/client.go | 147 +++++++++++++++---------------------- resources/plugin/plugin.go | 49 ------------- resources/plugin/tables.go | 128 ++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 136 deletions(-) create mode 100644 resources/plugin/tables.go diff --git a/resources/plugin/client.go b/resources/plugin/client.go index e5de03b..6612fa9 100644 --- a/resources/plugin/client.go +++ b/resources/plugin/client.go @@ -4,126 +4,99 @@ import ( "context" "fmt" + "github.com/cloudquery/plugin-sdk/v4/message" "github.com/cloudquery/plugin-sdk/v4/plugin" "github.com/cloudquery/plugin-sdk/v4/scheduler" "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/cloudquery/plugin-sdk/v4/transformers" "github.com/koltyakov/cq-source-sharepoint/resources/auth" - "github.com/koltyakov/cq-source-sharepoint/resources/services/ct" - "github.com/koltyakov/cq-source-sharepoint/resources/services/lists" - "github.com/koltyakov/cq-source-sharepoint/resources/services/mmd" - "github.com/koltyakov/cq-source-sharepoint/resources/services/profiles" - "github.com/koltyakov/cq-source-sharepoint/resources/services/search" "github.com/rs/zerolog" ) type Client struct { - lists *lists.Lists - mmd *mmd.MMD - profiles *profiles.Profiles - search *search.Search - contentTypes *ct.ContentTypesRollup + logger zerolog.Logger + spec Spec + tables schema.Tables + scheduler *scheduler.Scheduler + + options plugin.NewClientOptions + + plugin.UnimplementedDestination } -func NewClient(ctx context.Context, logger zerolog.Logger, cnfg []byte, opts plugin.NewClientOptions) (plugin.Client, error) { - spec, err := getSpec(cnfg) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal spec: %w", err) +func (*Client) ID() string { + return Name +} +func (c *Client) Sync(ctx context.Context, options plugin.SyncOptions, res chan<- message.SyncMessage) error { + if c.options.NoConnection { + return fmt.Errorf("no connection") } - sp, err := auth.GetSP(spec.Auth) + tt, err := c.Tables(ctx, plugin.TableOptions{ + Tables: options.Tables, + SkipTables: options.SkipTables, + SkipDependentTables: options.SkipDependentTables, + }) + if err != nil { - return nil, err + return err } - if _, err := sp.Web().Select("Title").Get(); err != nil { - return nil, fmt.Errorf("failed to connect to SharePoint: %w", err) - } + return c.scheduler.Sync(ctx, c, tt, res, scheduler.WithSyncDeterministicCQID(options.DeterministicCQID)) +} - client := &Client{ - lists: lists.NewLists(sp, logger), - mmd: mmd.NewMMD(sp, logger), - profiles: profiles.NewProfiles(sp, logger), - search: search.NewSearch(sp, logger), - contentTypes: ct.NewContentTypesRollup(sp, logger), +func (c *Client) Tables(_ context.Context, options plugin.TableOptions) (schema.Tables, error) { + if c.options.NoConnection { + return schema.Tables{}, nil } - tables, err := client.getTables(spec) + tt, err := c.tables.FilterDfs(options.Tables, options.SkipTables, options.SkipDependentTables) if err != nil { - return nil, fmt.Errorf("failed to retrieve tables: %w", err) - } - - if opts.NoConnection { - return &Plugin{ - logger: logger, - tables: tables, - }, nil + return nil, err } - return &Plugin{ - logger: logger, - spec: *spec, - tables: tables, - scheduler: scheduler.NewScheduler(scheduler.WithLogger(logger)), - client: client, - }, nil + return tt, nil } -func (c *Client) getTables(config *Spec) (schema.Tables, error) { - tables := schema.Tables{} - - // Tables from lists config - for listURI, listSpec := range config.Lists { - table, err := c.lists.GetDestTable(listURI, listSpec) - if err != nil { - return nil, fmt.Errorf("failed to get list '%s': %w", listURI, err) - } - tables = append(tables, table) - } +func (*Client) Close(context.Context) error { + // ToDo: Add your client cleanup here + return nil +} - // Tables from mmd config - for terSetID, mmdSpec := range config.MMD { - table, err := c.mmd.GetDestTable(terSetID, mmdSpec) - if err != nil { - return nil, fmt.Errorf("failed to get term set '%s': %w", terSetID, err) - } - tables = append(tables, table) - } +func NewClient(_ context.Context, logger zerolog.Logger, cnfg []byte, opts plugin.NewClientOptions) (plugin.Client, error) { + logger = logger.With().Str("plugin", "sharepoint").Logger() - // Tables from profiles config - if config.Profiles.Enabled { - table, err := c.profiles.GetDestTable(config.Profiles) - if err != nil { - return nil, fmt.Errorf("failed to get profiles: %w", err) - } - tables = append(tables, table) + if opts.NoConnection { + // no spec could be present + return &Client{ + logger: logger, + options: opts, + }, nil } - // Tables from search config - for searchName, searchSpec := range config.Search { - table, err := c.search.GetDestTable(searchName, searchSpec) - if err != nil { - return nil, fmt.Errorf("failed to get search '%s': %w", searchName, err) - } - tables = append(tables, table) + spec, err := getSpec(cnfg) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal spec: %w", err) } - // Tables from content types config - for ctName, ctSpec := range config.ContentTypes { - table, err := c.contentTypes.GetDestTable(ctName, ctSpec) - if err != nil { - return nil, fmt.Errorf("failed to get content type '%s': %w", ctName, err) - } - tables = append(tables, table) + sp, err := auth.GetSP(spec.Auth) + if err != nil { + return nil, err } - if err := transformers.TransformTables(tables); err != nil { - return nil, err + if _, err := sp.Web().Select("Title").Get(); err != nil { + return nil, fmt.Errorf("failed to connect to SharePoint: %w", err) } - for _, table := range tables { - schema.AddCqIDs(table) + tables, err := spec.getTables(sp, logger) + if err != nil { + return nil, fmt.Errorf("failed to retrieve tables: %w", err) } - return tables, nil + return &Client{ + logger: logger, + spec: *spec, + tables: tables, + scheduler: scheduler.NewScheduler(scheduler.WithLogger(logger)), + options: opts, + }, nil } diff --git a/resources/plugin/plugin.go b/resources/plugin/plugin.go index d48b612..908078e 100644 --- a/resources/plugin/plugin.go +++ b/resources/plugin/plugin.go @@ -1,13 +1,7 @@ package plugin import ( - "context" - - "github.com/cloudquery/plugin-sdk/v4/message" "github.com/cloudquery/plugin-sdk/v4/plugin" - "github.com/cloudquery/plugin-sdk/v4/scheduler" - "github.com/cloudquery/plugin-sdk/v4/schema" - "github.com/rs/zerolog" ) var ( @@ -17,49 +11,6 @@ var ( Version = "development" ) -type Plugin struct { - logger zerolog.Logger - spec Spec - tables schema.Tables - scheduler *scheduler.Scheduler - - client *Client - - plugin.UnimplementedDestination -} - func NewPlugin() *plugin.Plugin { return plugin.NewPlugin(Name, Version, NewClient, plugin.WithKind(Kind), plugin.WithTeam(Team)) } - -func (*Plugin) ID() string { - return Name -} - -func (p *Plugin) Sync(ctx context.Context, options plugin.SyncOptions, res chan<- message.SyncMessage) error { - tt, err := p.Tables(ctx, plugin.TableOptions{ - Tables: options.Tables, - SkipTables: options.SkipTables, - SkipDependentTables: options.SkipDependentTables, - }) - - if err != nil { - return err - } - - return p.scheduler.Sync(ctx, p, tt, res, scheduler.WithSyncDeterministicCQID(options.DeterministicCQID)) -} - -func (p *Plugin) Tables(ctx context.Context, options plugin.TableOptions) (schema.Tables, error) { - tt, err := p.tables.FilterDfs(options.Tables, options.SkipTables, options.SkipDependentTables) - if err != nil { - return nil, err - } - - return tt, nil -} - -func (*Plugin) Close(ctx context.Context) error { - // ToDo: Add your client cleanup here - return nil -} diff --git a/resources/plugin/tables.go b/resources/plugin/tables.go new file mode 100644 index 0000000..3a4fd6f --- /dev/null +++ b/resources/plugin/tables.go @@ -0,0 +1,128 @@ +package plugin + +import ( + "fmt" + + "github.com/cloudquery/plugin-sdk/v4/schema" + "github.com/cloudquery/plugin-sdk/v4/transformers" + "github.com/koltyakov/cq-source-sharepoint/resources/services/ct" + "github.com/koltyakov/cq-source-sharepoint/resources/services/lists" + "github.com/koltyakov/cq-source-sharepoint/resources/services/mmd" + "github.com/koltyakov/cq-source-sharepoint/resources/services/profiles" + "github.com/koltyakov/cq-source-sharepoint/resources/services/search" + "github.com/koltyakov/gosip/api" + "github.com/rs/zerolog" +) + +func (s *Spec) getTables(sp *api.SP, logger zerolog.Logger) (schema.Tables, error) { + tables := schema.Tables{} + + // Tables from lists config + listTables, err := s.getListsTables(sp, logger) + if err != nil { + return nil, err + } + tables = append(tables, listTables...) + + // Tables from mmd config + mmdTables, err := s.getMMDTables(sp, logger) + if err != nil { + return nil, err + } + tables = append(tables, mmdTables...) + + // Tables from profiles config + profileTables, err := s.getProfileTables(sp, logger) + if err != nil { + return nil, err + } + tables = append(tables, profileTables...) + + // Tables from search config + searchTables, err := s.getSearchTables(sp, logger) + if err != nil { + return nil, err + } + tables = append(tables, searchTables...) + + // Tables from content types config + ctTables, err := s.getContentTypeTables(sp, logger) + if err != nil { + return nil, err + } + tables = append(tables, ctTables...) + + if err := transformers.TransformTables(tables); err != nil { + return nil, err + } + + for _, table := range tables { + schema.AddCqIDs(table) + } + + return tables, nil +} + +func (s *Spec) getListsTables(sp *api.SP, logger zerolog.Logger) (schema.Tables, error) { + tables := make(schema.Tables, 0, len(s.Lists)) + l := lists.NewLists(sp, logger) + for uri, spec := range s.Lists { + table, err := l.GetDestTable(uri, spec) + if err != nil { + return nil, fmt.Errorf("failed to get list '%s': %w", uri, err) + } + tables = append(tables, table) + } + return tables, nil +} + +func (s *Spec) getMMDTables(sp *api.SP, logger zerolog.Logger) (schema.Tables, error) { + tables := make(schema.Tables, 0, len(s.MMD)) + m := mmd.NewMMD(sp, logger) + for id, spec := range s.MMD { + table, err := m.GetDestTable(id, spec) + if err != nil { + return nil, fmt.Errorf("failed to get term set '%s': %w", id, err) + } + tables = append(tables, table) + } + return tables, nil +} + +func (s *Spec) getProfileTables(sp *api.SP, logger zerolog.Logger) (schema.Tables, error) { + if !s.Profiles.Enabled { + return nil, nil + } + + table, err := profiles.NewProfiles(sp, logger).GetDestTable(s.Profiles) + if err != nil { + return nil, fmt.Errorf("failed to get profiles: %w", err) + } + return schema.Tables{table}, nil +} + +func (s *Spec) getSearchTables(sp *api.SP, logger zerolog.Logger) (schema.Tables, error) { + tables := make(schema.Tables, 0, len(s.Search)) + srch := search.NewSearch(sp, logger) + for name, spec := range s.Search { + table, err := srch.GetDestTable(name, spec) + if err != nil { + return nil, fmt.Errorf("failed to get search '%s': %w", name, err) + } + tables = append(tables, table) + } + return tables, nil +} + +func (s *Spec) getContentTypeTables(sp *api.SP, logger zerolog.Logger) (schema.Tables, error) { + tables := make(schema.Tables, 0, len(s.ContentTypes)) + c := ct.NewContentTypesRollup(sp, logger) + for name, spec := range s.ContentTypes { + table, err := c.GetDestTable(name, spec) + if err != nil { + return nil, fmt.Errorf("failed to get content type '%s': %w", name, err) + } + tables = append(tables, table) + } + return tables, nil +} From f54fc5bbe23f8301ec46287967fa74c0fc6e052a Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Tue, 31 Oct 2023 18:52:11 +0200 Subject: [PATCH 2/2] package --- Makefile | 6 +++++- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 2743370..13b0941 100644 --- a/Makefile +++ b/Makefile @@ -15,4 +15,8 @@ fmt: .PHONY: build build: - go build -o bin/cq-source-sharepoint -v \ No newline at end of file + go build -o bin/cq-source-sharepoint -v + +.PHONY: package +package: + go run main.go package --docs-dir docs -m @CHANGELOG.md v2.0.0 . \ No newline at end of file diff --git a/go.mod b/go.mod index 0df5b96..bb6e9e0 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/AlecAivazis/survey/v2 v2.3.7 github.com/apache/arrow/go/v14 v14.0.0-20231030205031-cb11e44d878f github.com/brianvoe/gofakeit/v6 v6.24.0 - github.com/cloudquery/plugin-sdk/v4 v4.17.0 + github.com/cloudquery/plugin-sdk/v4 v4.17.1 github.com/google/uuid v1.4.0 github.com/koltyakov/gosip v0.0.0-20231003001958-007c8072d71c github.com/koltyakov/gosip-sandbox v0.0.0-20230410140555-1211f873b91c diff --git a/go.sum b/go.sum index ca6807c..be87a6b 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,8 @@ github.com/cloudquery/plugin-pb-go v1.13.1 h1:UR07rJgiExsY6TSDNvSHyaYsZl/QSIK62I github.com/cloudquery/plugin-pb-go v1.13.1/go.mod h1:dpnHh8INCc+TYrOCHFKmnEFjerTrpIHCJ3u9NRGB2h8= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug= -github.com/cloudquery/plugin-sdk/v4 v4.17.0 h1:R+6M4Unf+zkhwW6nOvhqRLeNUCF0Cos+CvkieFRNM2A= -github.com/cloudquery/plugin-sdk/v4 v4.17.0/go.mod h1:vGiAHFS2sOodpk1NC8rwuYjRp53oFuEk2mEq4MXgAJc= +github.com/cloudquery/plugin-sdk/v4 v4.17.1 h1:BQkDpWThRfqq5jKld9r7FAwfoXHV3+kMqaWTO+Wr//M= +github.com/cloudquery/plugin-sdk/v4 v4.17.1/go.mod h1:vGiAHFS2sOodpk1NC8rwuYjRp53oFuEk2mEq4MXgAJc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI=