diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..f996ea36 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +vendor/* +!vendor/vendor.json \ No newline at end of file diff --git a/.secrets.baseline b/.secrets.baseline index ba4eaeea..04fd2ffe 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2021-03-04T21:19:28Z", + "generated_at": "2021-04-16T18:38:00Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -528,15 +528,7 @@ "verified_result": null }, { - "hashed_secret": "863a5eb549736efd243d35b1f9d925bbd925560e", - "is_secret": false, - "is_verified": false, - "line_number": 306, - "type": "Base64 High Entropy String", - "verified_result": null - }, - { - "hashed_secret": "ec6efd2d1f9e804335b454e87f32f8d872430a02", + "hashed_secret": "8072f323c55a6892918a692e58cd0ac31bc7063b", "is_secret": false, "is_verified": false, "line_number": 308, @@ -545,7 +537,7 @@ } ] }, - "version": "0.13.1+ibm.31.dss", + "version": "0.13.1+ibm.34.dss", "word_list": { "file": null, "hash": null diff --git a/bluemix/version.go b/bluemix/version.go index b05afec8..7351bf39 100644 --- a/bluemix/version.go +++ b/bluemix/version.go @@ -3,7 +3,7 @@ package bluemix import "fmt" // Version is the SDK version -var Version = VersionType{Major: 0, Minor: 5, Build: 0} +var Version = VersionType{Major: 0, Minor: 5, Build: 2} // VersionType describe version info type VersionType struct { diff --git a/common/rest/client.go b/common/rest/client.go index 05da8f4b..d1f89905 100644 --- a/common/rest/client.go +++ b/common/rest/client.go @@ -9,10 +9,15 @@ import ( "io" "io/ioutil" "net/http" + + "gopkg.in/yaml.v2" + + . "github.com/IBM-Cloud/ibm-cloud-cli-sdk/common/rest/helpers" ) // ErrEmptyResponseBody means the client receives an unexpected empty response from server var ErrEmptyResponseBody = errors.New("empty response body") +var bufferSize = 1024 // ErrorResponse is the status code and response received from the server when an error occurs. type ErrorResponse struct { @@ -84,7 +89,15 @@ func (c *Client) DoWithContext(ctx context.Context, r *Request, respV interface{ case io.Writer: _, err = io.Copy(respV.(io.Writer), resp.Body) default: - err = json.NewDecoder(resp.Body).Decode(respV) + // Determine the response type and the decoder that should be used. + // If buffer is identified as json, use JSON decoder, otherwise + // assume the buffer contains yaml bytes + body, isJSON := IsJSONStream(resp.Body, bufferSize) + if isJSON { + err = json.NewDecoder(body).Decode(respV) + } else { + err = yaml.NewDecoder(body).Decode(respV) + } if err == io.EOF { err = ErrEmptyResponseBody } diff --git a/common/rest/client_test.go b/common/rest/client_test.go index 9209b52d..6a30e58e 100644 --- a/common/rest/client_test.go +++ b/common/rest/client_test.go @@ -35,6 +35,48 @@ func TestDoWithContext_WithSuccessV(t *testing.T) { assert.Equal("bar", res["foo"]) } +func TestDoWithContext_WithSuccessYAML(t *testing.T) { + assert := assert.New(t) + + ts := httptest.NewServer(serveHandler(200, "\"foo\": \"bar\"")) + defer ts.Close() + + var res map[string]string + resp, err := NewClient().DoWithContext(context.TODO(), GetRequest(ts.URL), &res, nil) + assert.NoError(err) + assert.Equal(http.StatusOK, resp.StatusCode) + assert.Equal("bar", res["foo"]) +} + +func TestDoWithContext_WithSuccessComplexYAML(t *testing.T) { + assert := assert.New(t) + + ts := httptest.NewServer(serveHandler(200, ` +--- +doe: "a deer, a female deer" +ray: "a drop of golden sun" +pi: 3.14159 +xmas: true +french-hens: 3 +calling-birds: + - huey + - dewey + - louie + - fred`)) + defer ts.Close() + + var res map[string]interface{} + resp, err := NewClient().DoWithContext(context.TODO(), GetRequest(ts.URL), &res, nil) + assert.NoError(err) + assert.Equal(http.StatusOK, resp.StatusCode) + assert.Equal("a deer, a female deer", res["doe"]) + assert.Equal("a drop of golden sun", res["ray"]) + assert.Equal(3.14159, res["pi"]) + assert.Equal(true, res["xmas"]) + callingBirds := []interface{}{"huey", "dewey", "louie", "fred"} + assert.Equal(callingBirds, res["calling-birds"]) +} + func TestDoWithContext_NilContext_WithSuccessV(t *testing.T) { assert := assert.New(t) @@ -48,6 +90,19 @@ func TestDoWithContext_NilContext_WithSuccessV(t *testing.T) { assert.Equal("bar", res["foo"]) } +func TestDoWithContext_NilContext_WithSuccessYAML(t *testing.T) { + assert := assert.New(t) + + ts := httptest.NewServer(serveHandler(200, "\"foo\": \"bar\"")) + defer ts.Close() + + var res map[string]string + resp, err := NewClient().DoWithContext(nil, GetRequest(ts.URL), &res, nil) + assert.NoError(err) + assert.Equal(http.StatusOK, resp.StatusCode) + assert.Equal("bar", res["foo"]) +} + func TestDoWithContext_WithoutSuccessV(t *testing.T) { assert := assert.New(t) @@ -72,6 +127,19 @@ func TestDo_WithSuccessV(t *testing.T) { assert.Equal("bar", res["foo"]) } +func TestDo_WithSuccessYAML(t *testing.T) { + assert := assert.New(t) + + ts := httptest.NewServer(serveHandler(200, "\"foo\": \"bar\"")) + defer ts.Close() + + var res map[string]string + resp, err := NewClient().Do(GetRequest(ts.URL), &res, nil) + assert.NoError(err) + assert.Equal(http.StatusOK, resp.StatusCode) + assert.Equal("bar", res["foo"]) +} + func TestDo_WithoutSuccessV(t *testing.T) { assert := assert.New(t) diff --git a/common/rest/helpers/client_helpers.go b/common/rest/helpers/client_helpers.go new file mode 100644 index 00000000..3a95f4da --- /dev/null +++ b/common/rest/helpers/client_helpers.go @@ -0,0 +1,36 @@ +package rest + +import ( + "bufio" + "bytes" + "io" + "unicode" +) + +var jsonPrefix = []byte("{") +var jsonArrayPrefix = []byte("[") + +// IsJSONStream scans the provided reader up to size, looking +// for a json prefix indicating this is JSON. It will return the +// bufio.Reader it creates for the consumer. The buffer size will at either be the size of 'size' +// or the size of the Reader passed in 'r', whichever is larger. +// Inspired from https://github.com/kubernetes/apimachinery/blob/v0.21.0/pkg/util/yaml/decoder.go +func IsJSONStream(r io.Reader, size int) (io.Reader, bool) { + buffer := bufio.NewReaderSize(r, size) + b, _ := buffer.Peek(size) + return buffer, containsJSON(b) +} + +// containsJSON returns true if the provided buffer appears to start with +// a JSON open brace or JSON open bracket. +// Inspired from https://github.com/kubernetes/apimachinery/blob/v0.21.0/pkg/util/yaml/decoder.go +func containsJSON(buf []byte) bool { + return hasPrefix(buf, jsonPrefix) || hasPrefix(buf, jsonArrayPrefix) +} + +// Return true if the first non-whitespace bytes in buf is +// prefix. +func hasPrefix(buf []byte, prefix []byte) bool { + trim := bytes.TrimLeftFunc(buf, unicode.IsSpace) + return bytes.HasPrefix(trim, prefix) +} diff --git a/common/rest/helpers/client_helpers_test.go b/common/rest/helpers/client_helpers_test.go new file mode 100644 index 00000000..54e6a4e5 --- /dev/null +++ b/common/rest/helpers/client_helpers_test.go @@ -0,0 +1,198 @@ +package rest + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" +) + +var chunkSize int + +func TestIsJSONStreamWithJSON(t *testing.T) { + assert := assert.New(t) + + chunkSize = 1024 + jsonBytes := []byte(`{ + "foo": { + "bar": 1 + "otherStuff": 2 + } + }`) + jsonBytesAsString := string(jsonBytes) + + r, isJSON := IsJSONStream(bytes.NewReader(jsonBytes), chunkSize) + + assert.NotNil(r) + + raw, _ := ioutil.ReadAll(r) + + assert.NotNil(raw) + assert.True(isJSON) + assert.Equal(jsonBytesAsString, string(raw)) +} + +func TestIsJSONStreamWithJSONArray(t *testing.T) { + assert := assert.New(t) + + chunkSize = 1024 + jsonBytes := []byte(`[ + { + "foo": { + "bar": 1 + "otherStuff": 2 + } + }, + { + "foo2": { + "bar": 1 + "otherStuff": 2 + } + } + ]`) + jsonBytesAsString := string(jsonBytes) + + r, isJSON := IsJSONStream(bytes.NewReader(jsonBytes), chunkSize) + + assert.NotNil(r) + + raw, _ := ioutil.ReadAll(r) + + assert.NotNil(raw) + assert.True(isJSON) + assert.Equal(jsonBytesAsString, string(raw)) +} + +func TestIsJSONStreamWithSpaceJSON(t *testing.T) { + assert := assert.New(t) + + chunkSize = 1024 + jsonBytes := []byte("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + + "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + + "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\t\t\t\t\t\t\t\t\t" + + "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" + + "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" + + "\t\t\t\t\t\t\t\t\t\t{\"foo\": \"bar\"}") + jsonBytesAsString := string(jsonBytes) + + r, isJSON := IsJSONStream(bytes.NewReader(jsonBytes), chunkSize) + + assert.NotNil(r) + + raw, _ := ioutil.ReadAll(r) + + assert.NotNil(raw) + assert.True(isJSON) + assert.Equal(jsonBytesAsString, string(raw)) +} + +func TestIsJSONStreamWithInvalidJSON(t *testing.T) { + assert := assert.New(t) + + chunkSize = 1024 + jsonBytes := []byte(` + "foo": { + "bar": 1 + "otherStuff": 2 + } + `) + jsonBytesAsString := string(jsonBytes) + + r, isJSON := IsJSONStream(bytes.NewReader(jsonBytes), chunkSize) + + assert.NotNil(r) + + raw, _ := ioutil.ReadAll(r) + + assert.NotNil(raw) + assert.False(isJSON) + assert.Equal(jsonBytesAsString, string(raw)) +} + +func TestIsJSONStreamWithString(t *testing.T) { + assert := assert.New(t) + + chunkSize = 1024 + jsonBytes := []byte("foobar") + jsonBytesAsString := string(jsonBytes) + + r, isJSON := IsJSONStream(bytes.NewReader(jsonBytes), chunkSize) + + assert.NotNil(r) + + raw, _ := ioutil.ReadAll(r) + + assert.NotNil(raw) + assert.False(isJSON) + assert.Equal(jsonBytesAsString, string(raw)) +} + +func TestIsJSONStreamWithYAML(t *testing.T) { + assert := assert.New(t) + + chunkSize = 1024 + jsonBytes := []byte(`--- + foo: + - bar: jon_snow + description: "The king of the north" + created: 2016-01-14 + updated: 2019-08-16 + company: IBM + `) + jsonBytesAsString := string(jsonBytes) + + r, isJSON := IsJSONStream(bytes.NewReader(jsonBytes), chunkSize) + + assert.NotNil(r) + + raw, _ := ioutil.ReadAll(r) + + assert.NotNil(raw) + assert.False(isJSON) + assert.Equal(jsonBytesAsString, string(raw)) +} + +func TestIsJSONStreamWithYamlArray(t *testing.T) { + assert := assert.New(t) + + chunkSize = 1024 + jsonBytes := []byte(` + items: + - things: + thing1: huey + things2: dewey + thing3: louie + - other things: + key: value + `) + jsonBytesAsString := string(jsonBytes) + + r, isJSON := IsJSONStream(bytes.NewReader(jsonBytes), chunkSize) + + assert.NotNil(r) + + raw, _ := ioutil.ReadAll(r) + + assert.NotNil(raw) + assert.False(isJSON) + assert.Equal(jsonBytesAsString, string(raw)) +} + +func TestIsJSONStreamWithEmptyString(t *testing.T) { + assert := assert.New(t) + + chunkSize = 1024 + jsonBytes := []byte("") + jsonBytesAsString := string(jsonBytes) + + r, isJSON := IsJSONStream(bytes.NewReader(jsonBytes), chunkSize) + + assert.NotNil(r) + + raw, _ := ioutil.ReadAll(r) + + assert.NotNil(raw) + assert.False(isJSON) + assert.Equal(jsonBytesAsString, string(raw)) +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 47342021..2ac04220 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -303,10 +303,10 @@ "revisionTime": "2017-05-08T13:24:47Z" }, { - "checksumSHA1": "0KwOlQV1dNUh9X8t+5s7nX5bqfk=", + "checksumSHA1": "RqcbcMbbS5iVjpckNxDc30/WYSE=", "path": "gopkg.in/yaml.v2", - "revision": "a3f3340b5840cee44f372bddb5880fcbc419b46a", - "revisionTime": "2017-02-08T14:18:51Z" + "revision": "7649d4548cb53a614db133b2a8ac1f31859dda8c", + "revisionTime": "2020-11-17T15:46:20Z" } ], "rootPath": "github.com/IBM-Cloud/ibm-cloud-cli-sdk"