pushstream: article content (long-)polling.

cached_ops.go

@@ -171,6 +171,11 @@ func generateArticle(key cache.Key) (cache.Cacheable, error) {
 			}
 			a.ContentTailHtml = buf.Bytes()
 		}
+		a.CacheKey = ptail.CacheKey
+		a.NextOffset = ptail.FileSize - TailSize + ptail.Offset + ptail.Length
+	} else {
+		a.CacheKey = p.CacheKey
+		a.NextOffset = p.Length
 	}
 
 	ar := article.NewRenderer()
@@ -186,6 +191,47 @@ func generateArticle(key cache.Key) (cache.Cacheable, error) {
 	return a, nil
 }
 
+type ArticlePartRequest struct {
+	Brd      pttbbs.Board
+	Filename string
+	CacheKey string
+	Offset   int
+}
+
+func (r *ArticlePartRequest) String() string {
+	return fmt.Sprintf("pttweb:bbs/%v/%v#%v,%v", r.Brd.BrdName, r.Filename, r.CacheKey, r.Offset)
+}
+
+func generateArticlePart(key cache.Key) (cache.Cacheable, error) {
+	r := key.(*ArticlePartRequest)
+
+	p, err := ptt.GetArticleSelect(r.Brd.Bid, pttbbs.SelectHead, r.Filename, r.CacheKey, r.Offset, -1)
+	if err == pttbbs.ErrNotFound {
+		// Returns an invalid result
+		return new(ArticlePart), nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	ap := new(ArticlePart)
+	ap.IsValid = true
+	ap.CacheKey = p.CacheKey
+	ap.NextOffset = r.Offset + p.Offset + p.Length
+
+	if len(p.Content) > 0 {
+		ar := article.NewRenderer()
+		ar.DisableArticleHeader = true
+		buf, err := ar.Render(p.Content)
+		if err != nil {
+			return nil, err
+		}
+		ap.ContentHtml = string(buf.Bytes())
+	}
+
+	return ap, nil
+}
+
 func truncateLargeContent(content []byte, size, maxScan int) []byte {
 	if len(content) <= size {
 		return content

config.go

@@ -22,6 +22,9 @@ type PttwebConfig struct {
 
 	FeedPrefix            string
 	AtomFeedTitleTemplate string
+
+	PushStreamSharedSecret      string
+	PushStreamSubscribeLocation string
 }
 
 const (

page/ajax.go

@@ -0,0 +1,17 @@
+package page
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+type ArticlePollResp struct {
+	ContentHtml string `json:"contentHtml"`
+	PollUrl     string `json:"pollUrl"`
+	Success     bool   `json:"success"`
+}
+
+func WriteAjaxResp(w http.ResponseWriter, obj interface{}) error {
+	w.Header().Set("Content-Type", "application/json")
+	return json.NewEncoder(w).Encode(obj)
+}

page/pages.go

@@ -86,6 +86,9 @@ type BbsArticle struct {
 	ContentHtml      string
 	ContentTailHtml  string
 	ContentTruncated bool
+	PollUrl          string
+	LongPollUrl      string
+	CurrOffset       int
 }
 
 func (BbsArticle) TemplateName() string { return TnameBbsArticle }

pttweb.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	_ "net/http/pprof"
 	"net/url"
 	"os"
 	"os/signal"
@@ -29,6 +30,7 @@ import (
 	"github.com/ptt/pttweb/page"
 	manpb "github.com/ptt/pttweb/proto/man"
 	"github.com/ptt/pttweb/pttbbs"
+	"github.com/ptt/pttweb/pushstream"
 
 	"github.com/gorilla/mux"
 )
@@ -41,6 +43,7 @@ const (
 
 var (
 	ErrOver18CookieNotEnabled = errors.New("board is over18 but cookie not enabled")
+	ErrSigMismatch            = errors.New("push stream signature mismatch")
 )
 
 var ptt pttbbs.Pttbbs
@@ -162,6 +165,7 @@ func createRouter() *mux.Router {
 	router.HandleFunc(`/atom/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}.xml`, errorWrapperHandler(handleBoardAtomFeed))
 	router.HandleFunc(`/bbs/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}/{filename:[MG]\.\d+\.A(?:\.[0-9A-F]+)?}.html`, errorWrapperHandler(handleArticle)).Name("bbsarticle")
 	router.HandleFunc(`/b/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}/{aidc:[0-9A-Za-z\-_]+}`, errorWrapperHandler(handleAidc)).Name("bbsaidc")
+	router.HandleFunc(`/poll/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}/{filename:[MG]\.\d+\.A(\.[0-9A-F]+)?}.html`, errorWrapperHandler(handleArticlePoll)).Name("bbsarticlepoll")
 	router.HandleFunc(`/ask/over18`, errorWrapperHandler(handleAskOver18)).Name("askover18")
 	router.HandleFunc(`/man/{fullpath:[0-9a-zA-Z_\.\-\/]+}.html`, errorWrapperHandler(handleMan)).Name("manentry")
 	return router
@@ -479,6 +483,11 @@ func handleArticleCommon(c *Context, w http.ResponseWriter, brdname, filename st
 		log.Println("Large rendered article:", brd.BrdName, filename, len(ar.ContentHtml))
 	}
 
+	pollUrl, longPollUrl, err := uriForPolling(brd.BrdName, filename, ar.CacheKey, ar.NextOffset)
+	if err != nil {
+		return err
+	}
+
 	return page.ExecutePage(w, &page.BbsArticle{
 		Title:            ar.ParsedTitle,
 		Description:      ar.PreviewContent,
@@ -487,6 +496,9 @@ func handleArticleCommon(c *Context, w http.ResponseWriter, brdname, filename st
 		ContentHtml:      string(ar.ContentHtml),
 		ContentTailHtml:  string(ar.ContentTailHtml),
 		ContentTruncated: ar.IsTruncated,
+		PollUrl:          pollUrl,
+		LongPollUrl:      longPollUrl,
+		CurrOffset:       ar.NextOffset,
 	})
 }
 
@@ -500,6 +512,86 @@ func oldFilename(filename string) (string, bool) {
 	return filename[:len(filename)-4], true
 }
 
+func verifySignature(brdname, filename string, size int64, sig string) bool {
+	return (&pushstream.PushNotification{
+		Brdname:   brdname,
+		Filename:  filename,
+		Size:      size,
+		Signature: sig,
+	}).CheckSignature(config.PushStreamSharedSecret)
+}
+
+func handleArticlePoll(c *Context, w http.ResponseWriter) error {
+	vars := mux.Vars(c.R)
+	brdname := vars["brdname"]
+	filename := vars["filename"]
+	cacheKey := c.R.FormValue("cacheKey")
+	offset, err := strconv.Atoi(c.R.FormValue("offset"))
+	if err != nil {
+		return err
+	}
+	size, err := strconv.Atoi(c.R.FormValue("size"))
+	if err != nil {
+		return err
+	}
+
+	if !verifySignature(brdname, filename, int64(offset), c.R.FormValue("offset-sig")) ||
+		!verifySignature(brdname, filename, int64(size), c.R.FormValue("size-sig")) {
+		return ErrSigMismatch
+	}
+
+	brd, err := getBoardByName(c, brdname)
+	if err != nil {
+		return err
+	}
+
+	obj, err := cacheMgr.Get(&ArticlePartRequest{
+		Brd:      *brd,
+		Filename: filename,
+		CacheKey: cacheKey,
+		Offset:   offset,
+	}, ZeroArticlePart, time.Second, generateArticlePart)
+	if err != nil {
+		return err
+	}
+	ap := obj.(*ArticlePart)
+
+	res := new(page.ArticlePollResp)
+	res.Success = ap.IsValid
+	if ap.IsValid {
+		res.ContentHtml = ap.ContentHtml
+		res.PollUrl, _, err = uriForPolling(brdname, filename, ap.CacheKey, ap.NextOffset)
+		if err != nil {
+			return err
+		}
+	}
+	return page.WriteAjaxResp(w, res)
+}
+
+func uriForPolling(brdname, filename, cacheKey string, offset int) (poll, longPoll string, err error) {
+	pn := pushstream.PushNotification{
+		Brdname:  brdname,
+		Filename: filename,
+		Size:     int64(offset),
+	}
+	pn.Sign(config.PushStreamSharedSecret)
+
+	next, err := router.Get("bbsarticlepoll").URLPath("brdname", brdname, "filename", filename)
+	if err != nil {
+		return
+	}
+	args := make(url.Values)
+	args.Set("cacheKey", cacheKey)
+	args.Set("offset", strconv.FormatInt(pn.Size, 10))
+	args.Set("offset-sig", pn.Signature)
+	poll = next.String() + "?" + args.Encode()
+
+	lpArgs := make(url.Values)
+	lpArgs.Set("id", pushstream.GetPushChannel(&pn, config.PushStreamSharedSecret))
+	longPoll = config.PushStreamSubscribeLocation + "?" + lpArgs.Encode()
+	return
+}
+
 func getBoardByName(c *Context, brdname string) (*pttbbs.Board, error) {
 	if !pttbbs.IsValidBrdName(brdname) {
 		return nil, NewNotFoundError(fmt.Errorf("invalid board name: %s", brdname))

pushstream/stream.go

@@ -0,0 +1,33 @@
+package pushstream
+
+import (
+	"crypto/sha1"
+	"fmt"
+)
+
+type PushNotification struct {
+	Brdname   string `json:"brdname"`
+	Filename  string `json:"filename"`
+	Size      int64  `json:"size"`
+	Signature string `json:"sig"`
+}
+
+func (p *PushNotification) Sign(secret string) {
+	p.Signature = p.calcSig(secret)
+}
+
+func (p *PushNotification) CheckSignature(secret string) bool {
+	return p.Signature == p.calcSig(secret)
+}
+
+func (p *PushNotification) calcSig(secret string) string {
+	return sha1hex(fmt.Sprintf("%v/%v/%v/%v", p.Brdname, p.Filename, p.Size, secret))
+}
+
+func GetPushChannel(p *PushNotification, secret string) string {
+	return sha1hex(fmt.Sprintf("%v/%v/%v", p.Brdname, p.Filename, secret))
+}
+
+func sha1hex(s string) string {
+	return fmt.Sprintf("%x", sha1.Sum([]byte(s)))
+}

struct.go

@@ -13,6 +13,7 @@ import (
 // Useful when calling |NewFromBytes|
 var (
 	ZeroArticle       *Article
+	ZeroArticlePart   *ArticlePart
 	ZeroBbsIndex      *BbsIndex
 	ZeroBoardAtomFeed *BoardAtomFeed
 )
@@ -45,6 +46,9 @@ type Article struct {
 	IsPartial       bool
 	IsTruncated     bool
 
+	CacheKey   string
+	NextOffset int
+
 	IsValid bool
 }
 
@@ -56,6 +60,21 @@ func (a *Article) EncodeToBytes() ([]byte, error) {
 	return gobEncodeBytes(a)
 }
 
+type ArticlePart struct {
+	ContentHtml string
+	CacheKey    string
+	NextOffset  int
+	IsValid     bool
+}
+
+func (_ *ArticlePart) NewFromBytes(data []byte) (cache.Cacheable, error) {
+	return gobDecodeCacheable(data, new(ArticlePart))
+}
+
+func (a *ArticlePart) EncodeToBytes() ([]byte, error) {
+	return gobEncodeBytes(a)
+}
+
 type BbsIndex page.BbsIndex
 
 func (_ *BbsIndex) NewFromBytes(data []byte) (cache.Cacheable, error) {
@@ -81,11 +100,13 @@ func (bi *BoardAtomFeed) EncodeToBytes() ([]byte, error) {
 
 func init() {
 	gob.Register(Article{})
+	gob.Register(ArticlePart{})
 	gob.Register(BbsIndex{})
 	gob.Register(BoardAtomFeed{})
 
 	// Make sure they are |Cacheable|
 	checkCacheable(new(Article))
+	checkCacheable(new(ArticlePart))
 	checkCacheable(new(BbsIndex))
 	checkCacheable(new(BoardAtomFeed))
 }