Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Http post #10

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ jobs:
run: make vet

- name: Test
run: make test

- name: Test-Cover
run: make test-cover

- name: Upload coverage
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ test: internal/sqlite/sqlite3.c
@CGO_LDFLAGS="${CGO_LDFLAGS}" go test -v -tags="libsqlite3,sqlite_json1" ./...

test-cover:
@CGO_LDFLAGS="${CGO_LDFLAGS}" go test -v -tags=$(TAGS) ./... -cover -covermode=count -coverprofile=coverage.out
@CGO_LDFLAGS="${CGO_LDFLAGS}" go test -v -tags="libsqlite3,sqlite_json1"./... -cover -covermode=count -coverprofile=coverage.out
@CGO_LDFLAGS="${CGO_LDFLAGS}" go tool cover -html=coverage.out

vet: internal/sqlite/sqlite3.c
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.16

require (
github.com/augmentable-dev/vtab v0.0.0-20210328214525-c302d68997b8
github.com/jmoiron/sqlx v1.3.4
github.com/mattn/go-sqlite3 v1.14.6
go.riyazali.net/sqlite v0.0.0-20210326190148-448ec1ab2454
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@ github.com/augmentable-dev/vtab v0.0.0-20210328214525-c302d68997b8 h1:Dkj6EXcZhE
github.com/augmentable-dev/vtab v0.0.0-20210328214525-c302d68997b8/go.mod h1:BKbu+b2ssBzOOxr8F3X1cVZa5K+cd452Ogc3qgcfIfY=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w=
github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0=
github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc=
github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package readfile
package file_read

import (
"io/ioutil"
Expand Down
32 changes: 20 additions & 12 deletions internal/http/get.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,40 +8,48 @@ import (
"go.riyazali.net/sqlite"
)

type get struct{}
type get struct {
client *http.Client
}

// TODO add PUT and POST stuff

func (m *get) Args() int { return -1 }
func (m *get) Deterministic() bool { return false }
func (m *get) Apply(ctx *sqlite.Context, values ...sqlite.Value) {
func (f *get) Args() int { return -1 }
func (f *get) Deterministic() bool { return false }
func (f *get) Apply(ctx *sqlite.Context, values ...sqlite.Value) {
var (
request string
url string
headers [][]string
err error
contents []byte
response *http.Response
request *http.Request
)

if len(values) > 0 {
request = values[0].Text()
url = values[0].Text()
} else if len(values) > 1 {
heads := values[1].Text()
headers = ParseHeaders(heads)
} else {
err := errors.New("input a single url as the argument to http get")
err := errors.New("input a single url as the argument to http get or a url with headers")
ctx.ResultError(err)
}

response, err = http.Get(request)
request, err = HttpRequest(url, headers, "GET")
if err != nil {
ctx.ResultError(err)
}
response, err := f.client.Do(request)
if err != nil {
ctx.ResultError(err)
}
contents, err = ioutil.ReadAll(response.Body)
if err != nil {
ctx.ResultError(err)
}

ctx.ResultText(string(contents))
}

// NewHTTPGet returns a sqlite function for reading the contents of a file
func NewHTTPGet() sqlite.Function {
return &get{}
return &get{http.DefaultClient}
}
29 changes: 29 additions & 0 deletions internal/http/helpers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package http

import (
"net/http"
"strings"
)

func ParseHeaders(headers string) [][]string {
headerList := strings.Split(headers, "|")
var kvHeaders [][]string
for _, s := range headerList {
st := strings.Split(s, ":")
for i := range st {
st[i] = strings.TrimSpace(st[i])
}
kvHeaders = append(kvHeaders, st)
}
return kvHeaders
}
func HttpRequest(requestUrl string, headers [][]string, requestType string) (*http.Request, error) {
request, err := http.NewRequest(requestType, requestUrl, nil)
if err != nil {
return nil, err
}
for _, header := range headers {
request.Header.Add(header[0], header[1])
}
return request, nil
}
161 changes: 161 additions & 0 deletions internal/http/helpers_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
package http

import (
"bytes"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"

_ "github.com/augmentable-dev/flite/internal/sqlite"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
"go.riyazali.net/sqlite"
)

func TestHeaderParser(t *testing.T) {
header := "Content-Type: application/json |Accept: application/json|accept-encoding: application/gzip|x-api-key: 09187304915uqiyoewue90832174109732y6985132"
expected := [][]string{{"Content-Type", "application/json"}, {"Accept", "application/json"}, {"accept-encoding", "application/gzip"}, {"x-api-key", "09187304915uqiyoewue90832174109732y6985132"}}
parsed := ParseHeaders(header)
for i := range expected {
if expected[i][0] != parsed[i][0] || expected[i][1] != parsed[i][1] {
t.Fatalf("expected %s,%s got %s,%s", expected[i][0], expected[i][1], parsed[i][0], parsed[i][1])
}
}
}

type mockRoundTripper struct {
f func(*http.Request) (*http.Response, error)
}

func (m *mockRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
return m.f(req)
}
func newMockRoundTripper(f func(*http.Request) (*http.Response, error)) *mockRoundTripper {
return &mockRoundTripper{f: f}
}
func TestHTTPGet(t *testing.T) {
getFunc := NewHTTPGet()
f := getFunc.(*get)
url := "https://some-url.com/v1/some-endpoint.json"
body := "OK"

f.client.Transport = newMockRoundTripper(func(req *http.Request) (*http.Response, error) {
if req.URL.String() != url {
t.Fatalf("expected request URL: %s, got: %s", url, req.URL.String())
}
return &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewBufferString(body)),
Header: make(http.Header),
}, nil
})
sqlite.Register(func(api *sqlite.ExtensionApi) (sqlite.ErrorCode, error) {
if err := api.CreateFunction("http_get", getFunc); err != nil {
return sqlite.SQLITE_ERROR, err
}
return sqlite.SQLITE_OK, nil
})
db, err := sqlx.Open("sqlite3", ":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()

row := db.QueryRow("SELECT http_get($1)", url)
err = row.Err()
if err != nil {
t.Fatal(err)
}
var res string
err = row.Scan(&res)
if err != nil {
t.Fatal(err)
}
if res != body {
t.Fatalf("expected response: %s, got: %s", body, res)
}
}
func TestHTTPPost(t *testing.T) {
postFunc := NewHTTPpost()
f := postFunc.(*post)
url := "https://some-url.com/v1/some-endpoint.json"
body := "OK"
header := "Content-Type: application/json |Accept: application/json|accept-encoding: application/gzip|x-api-key: 09187304915uqiyoewue90832174109732y6985132"
expected := map[string][]string{"Content-Type": {"application/json"}, "Accept": {"application/json"}, "accept-encoding": {"application/gzip"}, "x-api-key": {"09187304915uqiyoewue90832174109732y6985132"}}
f.client.Transport = newMockRoundTripper(func(req *http.Request) (*http.Response, error) {
if req.URL.String() != url {
t.Fatalf("expected request URL: %s, got: %s", url, req.URL.String())
}
if req.Header.Get("Content-Type") != expected["Content-Type"][0] {
t.Fatalf("expected Header Content-Type: %s got: %s", expected["Content-type"], req.Header.Get("Content-Type"))
}
return &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewBufferString(body)),
Header: expected,
}, nil
})
sqlite.Register(func(api *sqlite.ExtensionApi) (sqlite.ErrorCode, error) {
if err := api.CreateFunction("http_post", postFunc); err != nil {
return sqlite.SQLITE_ERROR, err
}
return sqlite.SQLITE_OK, nil
})
db, err := sqlx.Open("sqlite3", ":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()

row := db.QueryRow("SELECT http_post($1,$2)", url, header)
err = row.Err()
if err != nil {
t.Fatal(err)
}
var res string
err = row.Scan(&res)
if err != nil {
t.Fatal(err)
}
if res != body {
t.Fatalf("expected response: %s, got: %s", body, res)
}
}
func TestHttpRequest(t *testing.T) {
method := "GET"
url := "http://api.citybik.es/v2/networks"
responseRecorder := httptest.NewRecorder()
headers := [][]string{{"Content-Type", "application/json"}}
req, err := HttpRequest(url, headers, method)
if err != nil {
t.Fatal(err)
}
handler := http.HandlerFunc(httpHandler)
handler.ServeHTTP(responseRecorder, req)
if status := responseRecorder.Code; status != http.StatusOK {
t.Fatal(err)
}
expected := `{"alive": true}`
if responseRecorder.Body.String() != expected {
t.Fatalf("received %s expected %s", responseRecorder.Body.String(), expected)
}

}
func httpHandler(w http.ResponseWriter, r *http.Request) {
println(r.URL.String())
if r.URL.String() != "http://api.citybik.es/v2/networks" {
w.Header().Set("Content-Type", "application/json")
io.WriteString(w, `{"response": "incorrect url"`)
return
}
if r.Header.Get("Content-Type") != "application/json" {
w.Header().Set("Content-Type", "application/json")
io.WriteString(w, `{"response": "incorrect header"`)
return
}
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
io.WriteString(w, `{"alive": true}`)
}
53 changes: 53 additions & 0 deletions internal/http/post.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package http

import (
"errors"
"io/ioutil"
"net/http"

"go.riyazali.net/sqlite"
)

type post struct {
client *http.Client
}

// TODO add PUT and POST stuff
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can remove this probably


func (f *post) Args() int { return -1 }
func (f *post) Deterministic() bool { return false }
func (f *post) Apply(ctx *sqlite.Context, values ...sqlite.Value) {
var (
url string
headers [][]string
err error
contents []byte
request *http.Request
)
if len(values) > 1 {
url = values[0].Text()
heads := values[1].Text()
headers = ParseHeaders(heads)
} else {
err := errors.New("input a url with headers as the argument to post")
ctx.ResultError(err)
}
request, err = HttpRequest(url, headers, "POST")
if err != nil {
ctx.ResultError(err)
}
response, err := f.client.Do(request)
if err != nil {
ctx.ResultError(err)
}
contents, err = ioutil.ReadAll(response.Body)
if err != nil {
ctx.ResultError(err)
}
ctx.ResultText(string(contents))
}

// NewHTTPpost returns a sqlite function for reading the contents of a file
func NewHTTPpost() sqlite.Function {
return &post{http.DefaultClient}
}
4 changes: 2 additions & 2 deletions pkg/ext/ext.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package ext

import (
"github.com/augmentable-dev/flite/internal/file_read"
"github.com/augmentable-dev/flite/internal/file_split"
"github.com/augmentable-dev/flite/internal/http"
"github.com/augmentable-dev/flite/internal/readfile"
"github.com/augmentable-dev/flite/internal/yaml"
_ "github.com/mattn/go-sqlite3"
"go.riyazali.net/sqlite"
Expand All @@ -16,7 +16,7 @@ func init() {
return sqlite.SQLITE_ERROR, err
}

if err := api.CreateFunction("readfile", readfile.NewReadFile()); err != nil {
if err := api.CreateFunction("file_read", file_read.NewReadFile()); err != nil {
return sqlite.SQLITE_ERROR, err
}

Expand Down