diff options
Diffstat (limited to 'desk/lib/fetch.hoon')
| -rw-r--r-- | desk/lib/fetch.hoon | 546 |
1 files changed, 546 insertions, 0 deletions
diff --git a/desk/lib/fetch.hoon b/desk/lib/fetch.hoon new file mode 100644 index 0000000..bd74deb --- /dev/null +++ b/desk/lib/fetch.hoon @@ -0,0 +1,546 @@ +/- *boke, tp=trill-post, tlonc=tlon-channels +/+ sr=sortug, plib=trill-utils, kaji, const=constants, wall +|_ [s=state =bowl:gall] ++* wal ~(. wall src.bowl) +++ bulk-post-by-pid +|= pids=(list pid:tp) ^- (list post:tp) + %+ roll pids |= [=pid:tp acc=(list post:tp)] + =/ post (get:gorm:tp feed.s pid) + ?~ post acc :_ acc u.post + +++ thread-children +=| start=@ud +|= ted=thread:tp ^- cpage:tp + =| i=@ud + =/ lim (add thread-page-size:const start) + =/ cp=cpage:tp [~ start thread-page-size:const] + =/ pids (flop replies.ted) + =/ r=cpage:tp + |- + ?~ pids cp + ?: (lth i start) $(pids t.pids, i +(i)) + ?: (gte i lim) cp + =/ post (get:gorm:tp feed.s i.pids) + ?~ post $(pids t.pids) + =/ np [u.post p.cp] + =. cp cp(p np, bot i) + $(pids t.pids, i +(i)) + + r(p (flop p.r)) + +++ older-children +|= [ted=thread:tp ind=@ud] ^- cpage:tp + %. ted + %* . thread-children start ind == + +++ newer-children +|= [ted=thread:tp ind=@ud] ^- cpage:tp + =/ lim thread-page-size:const + =/ start ?: (lte ind lim) 0 (sub ind lim) + %. ted + %* . thread-children start start == + +++ gated-post + |= =pid:tp ^- (unit post:tp) + =/ upost (get:gorm:tp feed.s pid) + ?~ upost ~ + ?: (post-filter:wal tags.u.upost) upost ~ +++ by-path + |= pat=path ^- (unit post:tp) + =/ pid (~(get by paths.s) pat) + ?~ pid ~ + (gated-post u.pid) +++ post + |= hash=@t ^- (unit post:tp) + =/ pid (dec:kaji hash pid:tp) + ?~ pid ~ (gated-post u.pid) +++ fn-by-pid +|= =pid:tp ^- (unit full-node:tp) + =/ post (gated-post pid) + ?~ post ~ + %- some (node-to-full:plib u.post feed.s) +++ fn-by-hash +|= hash=@t ^- (unit full-node:tp) + =/ pid (dec:kaji hash pid:tp) + ?~ pid ~ + =/ post (gated-post u.pid) + ?~ post ~ + %- some (node-to-full:plib u.post feed.s) +++ thread-by-hash + |= hash=@t ^- (unit [thread:tp post:tp]) + =/ pid (dec:kaji hash pid:tp) + ?~ pid ~ + =/ ted (get:torm:tp threads.s u.pid) + ?~ ted ~ + =/ post (gated-post u.pid) + ?~ post ~ + %- some [u.ted u.post] +++ op-by-hash + |= hash=@t ^- (unit [thread:tp full-node:tp]) + =/ pid (dec:kaji hash pid:tp) + ?~ pid ~ + =/ ted (get:torm:tp threads.s u.pid) + ?~ ted ~ + =/ post (gated-post u.pid) + ?~ post ~ + %- some :- u.ted (node-to-full:plib u.post feed.s) +++ op-by-pid + |= =pid:tp ^- (unit [thread:tp full-node:tp]) + =/ ted (get:torm:tp threads.s pid) + ?~ ted ~ + =/ post (gated-post pid) + ?~ post ~ + %- some :- u.ted (node-to-full:plib u.post feed.s) +++ thread + |= =path + ^- (unit thread:tp) + =/ pid (~(get by paths.s) path) + ?~ pid ~ + (get:torm:tp threads.s u.pid) +++ op-by-path :: blog posts or thread ops + |= =path + ^- (unit [thread:tp full-node:tp]) + =/ pid (~(get by paths.s) path) + ?~ pid ~ + =/ ted (get:torm:tp threads.s u.pid) + ?~ ted ~ + =/ poast (gated-post u.pid) + ?~ poast ~ + %- some :- u.ted (node-to-full:plib u.poast feed.s) +++ tag-search + |= q=@t ^- (list [@t count=@ud]) + %+ roll ~(tap by tags.s) |= [i=[tag=@t =pidmap] acc=(list [@t @ud])] + ?. (tag-filter:wal tag.i) acc + ?: .=('' tag.i) acc + ?. (cfind:sr q tag.i .n) acc :_ acc + :- tag.i (wyt:porm:tp pidmap.i) + +++ thread-page-no-wall +|= [r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + =/ l (tap:torm:tp threads.s) + =| =tpage:tp + |- + ?~ l tpage(p (flop p.tpage)) + ?: (gte count.tpage count.r) tpage(p (flop p.tpage)) + =/ t=thread:tp +.i.l + =. tpage (collect-thread-no-wall t tpage r f) + $(l t.l) +:: +++ thread-page-all +|= [r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + =/ l (tap:torm:tp threads.s) + =| =tpage:tp + |- + ?~ l tpage(p (flop p.tpage)) + ?: (gte count.tpage count.r) tpage(p (flop p.tpage)) + =/ t=thread:tp +.i.l + =. tpage (collect-thread t tpage r f) + $(l t.l) +++ active-thread-page-all +|= [r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + =/ l (tap:porm:tp active-threads.s) + =| =tpage:tp + |- + ?~ l tpage(p (flop p.tpage)) + ?: (gte count.tpage count.r) tpage(p (flop p.tpage)) + =/ =pid:tp +.i.l + =/ ut (get:torm:tp threads.s pid) + ?~ ut $(l t.l) + =/ t=thread:tp u.ut + =. tpage (collect-thread t tpage r f) + $(l t.l) + +++ thread-page-by-board + |= [cat=@tas r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + =/ tag-set (~(get by categories:const) cat) + ?~ tag-set [~ r] + %^ thread-page-by-tags ~(tap in u.tag-set) r f +++ active-thread-page-by-board + |= [cat=@tas r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + =/ tag-set (~(get by categories:const) cat) + ?~ tag-set [~ r] + %^ active-thread-page-by-tags ~(tap in u.tag-set) r f + +++ thread-page-by-tags +|= [tags=(list @t) r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + :: =. tags (tags:wal tags) + ~> %bout.[0 %filter-tags-first] + =/ pids %+ roll tags |= [t=@t acc=pidmap] + =/ pids (~(get by tags.s) t) + ?~ pids acc + (uni:porm:tp acc u.pids) + ?: .=((wyt:porm:tp pids) 0) [~ r] + =/ l (fetch-threads-2 (tap:porm:tp pids)) + =| =tpage:tp + |- + ?~ l tpage(p (flop p.tpage)) + ?: (gte count.tpage count.r) tpage(p (flop p.tpage)) + =/ t=thread:tp i.l + =. tpage (collect-thread t tpage r f) + $(l t.l) + +++ active-thread-page-by-tags +|= [tags=(list @t) r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + :: =. tags (tags:wal tags) + ~> %bout.[0 %filter-tags-first] + =/ pids %+ roll tags |= [t=@t acc=pidmap] + =/ pids (~(get by tags.s) t) + ?~ pids acc + (uni:porm:tp acc u.pids) + ?: .=((wyt:porm:tp pids) 0) [~ r] + =/ l (fetch-last-replies (tap:porm:tp pids)) + =| =tpage:tp + |- + ?~ l tpage(p (flop p.tpage)) + ?: (gte count.tpage count.r) tpage(p (flop p.tpage)) + =/ t=thread:tp i.l + =. tpage (collect-thread t tpage r f) + $(l t.l) + + + +++ thread-page-by-tags2 +|= [tags=(list @t) r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + =. tags (tags:wal tags) + ~> %bout.[0 %filter-threads-first] + =/ l (tap:torm:tp threads.s) + ?~ l [~ r] + =/ ff |= t=thread:tp ?= ^ (~(int in tags.t) (silt tags)) + =/ l=(list [pid:tp thread:tp]) l + =| =tpage:tp + |- + ?~ l tpage(p (flop p.tpage)) + ?: (gte count.tpage count.r) tpage(p (flop p.tpage)) + =/ t=thread:tp +.i.l + =. tpage (collect-thread t tpage r `ff) + $(l t.l) +:: +++ fetch-last-replies +|= lp=(list [pid:tp pid:tp]) ^- (list thread:tp) + :: %- flop + %- sort :_ |= [a=thread:tp b=thread:tp] + =/ last-a ?~ replies.a pid.a i.replies.a + =/ last-b ?~ replies.b pid.b i.replies.b + (gte id.last-a id.last-b) + %+ roll lp |= [[* =pid:tp] acc=(list thread:tp)] + =/ t (get:torm:tp threads.s pid) + ?~ t acc + [u.t acc] + +++ fetch-threads-2 +|= lp=(list [pid:tp pid:tp]) ^- (list thread:tp) + %- flop + %+ roll lp |= [[* =pid:tp] acc=(list thread:tp)] + =/ t (get:torm:tp threads.s pid) + ?~ t acc [u.t acc] +++ fetch-threads +|= lp=(list pid:tp) ^- (list thread:tp) + %- flop + %+ roll lp |= [=pid:tp acc=(list thread:tp)] + =/ t (get:torm:tp threads.s pid) + ?~ t acc [u.t acc] + +++ thread-page +|= [l=(list thread:tp) r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + =| =tpage:tp + |- + ?~ l tpage(p (flop p.tpage)) + ?: (gte count.tpage count.r) tpage(p (flop p.tpage)) + =/ t=thread:tp i.l + =. tpage (collect-thread t tpage r f) + $(l t.l) + +++ collect-thread +|= [t=thread:tp =tpage:tp r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + =/ id=@da id.pid.t + ?. (post-filter:wal tags.t) tpage + ?. (page-cond id r) tpage + ?: ?& ?=(^ f) + =/ fres (u.f t) ?!(fres) == tpage + :: :: + =/ oldest ?~ older.tpage `id + ?: (lth id u.older.tpage) `id older.tpage + =/ newest ?~ newer.tpage `id + ?: (gth id u.newer.tpage) `id newer.tpage + %= tpage + p [t p.tpage] + older oldest + newer newest + count +(count.tpage) + == +++ collect-thread-no-wall +|= [t=thread:tp =tpage:tp r=page-req:tp f=(unit tfilter:tp)] ^- tpage:tp + =/ id=@da id.pid.t + ?. (page-cond id r) tpage + ?: ?& ?=(^ f) + =/ fres (u.f t) ?!(fres) == tpage + :: :: + =/ oldest ?~ older.tpage `id + ?: (lth id u.older.tpage) `id older.tpage + =/ newest ?~ newer.tpage `id + ?: (gth id u.newer.tpage) `id newer.tpage + %= tpage + p [t p.tpage] + older oldest + newer newest + count +(count.tpage) + == + + + +++ check-age +|= [id=@da newer=cursor:tp older=cursor:tp] ^- ? + ?~ newer + ?~ older .y + (lth id u.older) + ?~ older + (gth id u.newer) + ?&((gth id u.newer) (lth id u.older)) + +++ page-cond +|= [id=@da r=page-req:tp] ^- ? +?~ newer.r + :: newest not bound + ?~ older.r + :: neither oldest or newest is bound + .y + :: oldest is bound, newest isn't + (lth id u.older.r) + :: newest bound + ?~ older.r + :: newest is bound, oldest isn't + (gth id u.newer.r) + :: both are bound + ?&((lth id u.older.r) (gth id u.newer.r)) +++ get-post-page + |= [req=page-req:tp filter=(unit $-(post:tp ?))] ^- spage:tp + :: ~> %bout.[0 %get-post-page-rec] + =/ from-bottom=? ?& ?=(%~ older.req) ?!(?=(%~ newer.req)) == + =/ l ?. from-bottom (tap:gorm:tp feed.s) (flop (tap:gorm:tp feed.s)) + :: =| max=@da :: need to define the max and minimum + :: =| min=@da + :: this is what we return + =| p=spage:tp + |- + ?~ l p :: end of iteration + =/ [[=ship =id:tp] =post:tp] i.l + ?. (page-cond id req) p :: exit condition + =/ ok ?~ filter .y (u.filter post) + ?. ok $(l t.l) + :: =/ max ?: (gth id max) id max + :: =/ min ?: ?|((lth id min) .=(min *@da)) id min + =/ oldest ?~ older.p `id + ?: (lth id u.older.p) `id older.p + =/ newest ?~ newer.p `id + ?: (gth id u.newer.p) `id newer.p + %= $ + l t.l + p.p [post p.p] + older.p oldest + newer.p newest + count.p +(count.p) + == +++ get-fn-page + |= [req=page-req:tp filter=(unit $-(post:tp ?))] ^- page:tp + :: ~> %bout.[0 %get-fn-page-rec] + =/ from-bottom=? ?& ?=(%~ older.req) ?!(?=(%~ newer.req)) == + =/ l ?. from-bottom (tap:gorm:tp feed.s) (flop (tap:gorm:tp feed.s)) + :: =| max=@da :: need to define the max and minimum + :: =| min=@da + :: this is what we return + =| p=page:tp + |- + ?~ l p :: end of iteration + =/ [[=ship =id:tp] =post:tp] i.l + ?. (page-cond id req) p :: exit condition + =/ ok ?~ filter .y (u.filter post) + ?. ok $(l t.l) + :: =/ max ?: (gth id max) id max + :: =/ min ?: ?|((lth id min) .=(min *@da)) id min + =/ oldest ?~ older.p `id + ?: (lth id u.older.p) `id older.p + =/ newest ?~ newer.p `id + ?: (gth id u.newer.p) `id newer.p + =/ fn (node-to-full:plib post feed.s) + %= $ + l t.l + p.p [fn p.p] + older.p oldest + newer.p newest + count.p +(count.p) + == + :: Search +++ get-chat-id +|= r=reference:tlonc ^- @da + ?- -.r + %post id.post.r + %reply id-post.r + == +++ search +|= [s=section query=@t tags=(list @t) r=page-req:tp] ^- search-res + ?: ?=(%chat s) :- %chat (search-chat query r) + :- %trill (search-dap s query r) + +++ sort-fn +|= p=(list [full-node:tp @t]) ^- (list [full-node:tp @t]) + %+ sort p |= [a=[full-node:tp @t] b=[full-node:tp @t]] (gth id.p.-.a id.p.-.b) +:: +++ collect-search +|= [sp=search-page query=@t =post:tp req=page-req:tp filter=(unit pfilter:tp)] +^- search-page + =/ id id.post + ?. (check-age id newer.req older.req) sp + ?. (post-filter:wal tags.post) sp + ?: ?& ?=(^ filter) + =/ fres (u.filter post) ?!(fres) == sp + =/ content-snip=@t (text-search-ind:plib query 150 post .n) + ?: .=(content-snip '') sp + =/ oldest ?~ older.sp `id + ?: (lth id u.older.sp) `id older.sp + =/ newest ?~ newer.sp `id + ?: (gth id u.newer.sp) `id newer.sp + =/ fn (node-to-full:plib post feed.s) + %= sp + res [[fn content-snip] res.sp] + older oldest + newer newest + == + +++ search-feed +|= [query=@t req=page-req:tp filter=(unit pfilter:tp)] ^- search-page + =/ bottom ?~ newer.req ~ `[~zod u.newer.req] + =/ top ?~ older.req ~ `[~zod u.older.req] + =/ subfeed (lot:gorm:tp feed.s top bottom) + =/ l (tap:gorm:tp subfeed) + (search-tap query l req filter) + +++ search-tap +|= [query=@t l=(list [=pid:tp =post:tp]) req=page-req:tp filter=(unit pfilter:tp)] + ^- search-page + =| p=search-page + |- + ?~ l p(res (sort-fn res.p)) + ?: (gte (lent res.p) count.req) p(res (sort-fn res.p)) + =/ =post:tp post.i.l + =. p (collect-search p query post req filter) + $(l t.l) + + +++ search-pids-full +|= [query=@t l=(list pid:tp) req=page-req:tp filter=(unit pfilter:tp)] ^- search-page + =| p=search-page + |- + ?~ l p(res (sort-fn res.p)) + ?: (gte (lent res.p) count.req) p(res (sort-fn res.p)) + =/ post (get:gorm:tp feed.s i.l) + ?~ post $(l t.l) + =. p (collect-search p query u.post req filter) + $(l t.l) + ++$ dpage [by-title=(list [full-node:tp @t]) by-content=(list [full-node:tp @t]) newer=cursor:tp older=cursor:tp] + +++ collect-thread-search +|= [dp=dpage query=@t t=thread:tp =post:tp req=page-req:tp f=(unit bfilter:tp)] ^- dpage + ?: ?& ?=(^ f) + =/ fres (u.f [t post]) ?!(fres) == dp + =/ id id.pid.t + ?. (page-cond id req) dp + ?. (post-filter:wal tags.post) dp + =/ title-match=? (cfind:sr query title.t .n) + ?: title-match + =/ fn (node-to-full:plib post feed.s) + =/ nc (update-cursors id older.dp newer.dp) + dp(by-title [[fn ''] by-title.dp], older -.nc, newer +.nc) + =/ content-snip=@t (text-search-ind:plib query 200 post .n) + ?. .=('' content-snip) + =/ fn (node-to-full:plib post feed.s) + =/ nc (update-cursors id older.dp newer.dp) + dp(by-content [[fn content-snip] by-content.dp], older -.nc, newer +.nc) + :: no match + dp +++ consolidate-search +|= d=dpage ^- search-page + :+ (weld (sort-fn by-title.d) (sort-fn by-content.d)) + newer.d older.d + +++ update-cursors +|= [id=@da older=cursor:tp newer=cursor:tp] ^- [older=cursor:tp newer=cursor:tp] + =/ o ?~ older `id + ?: (lth id u.older) `id older + =/ n ?~ newer `id + ?: (gth id u.newer) `id newer + [o n] +++ search-blog +|= [query=@t req=page-req:tp] ^- search-page + =/ pidm (~(get by tags.s) 'blog') + ?~ pidm *search-page + =/ l (tap:porm:tp u.pidm) + %- consolidate-search + =| p=dpage + |- + ?~ l p + =/ =pid:tp +.i.l + =/ count (add (lent by-title.p) (lent by-content.p)) + ?: (gte count count.req) p + =/ post (get:gorm:tp feed.s pid) + ?~ post $(l t.l) + =/ ted (get:torm:tp threads.s pid) + ?~ ted $(l t.l) + =. p (collect-thread-search p query u.ted u.post req ~) + $(l t.l) +:: +++ search-comments +|= [query=@t r=page-req:tp] ^- search-page + =/ pids %+ roll (tap:torm:tp threads.s) |= [[* thread:tp] acc=(list pid:tp)] + (weld acc replies) + (search-pids-full query pids r ~) + +++ search-threads +|= [query=@t req=page-req:tp] ^- search-page + =/ l (tap:torm:tp threads.s) + =/ filter |= [=thread:tp =post:tp] + ?& !(~(has in tags.post) 'blog') + !(~(has in tags.post) 'blog-comment') + == + %- consolidate-search + =| p=dpage + |- + ?~ l p + =/ t=thread:tp +.i.l + =/ count (add (lent by-title.p) (lent by-content.p)) + ?: (gte count count.req) p + =/ post (get:gorm:tp feed.s pid.t) + ?~ post $(l t.l) + =. p (collect-thread-search p query t u.post req `filter) + $(l t.l) +++ search-replies +|= [query=@t r=page-req:tp] ^- search-page + =/ filter |= =post:tp + ?& !(~(has in tags.post) 'blog') + !(~(has in tags.post) 'blog-comment') + ?=(^ parent.post) + == + (search-feed query r `filter) + +++ search-dap +|= [s=$?(%blog %comments %threads %replies) query=@t r=page-req:tp] ^- search-page + ?- s + %blog (search-blog query r) + %comments (search-comments query r) + %threads (search-threads query r) + %replies (search-replies query r) + == +++ search-chat +|= [query=@t req=page-req:tp] ^- chat-page + =/ flag /chat/(scot %p our.bowl)/chat + =/ pat /search/text/(scot %ud 0)/(scot %ud count.req)/[query] + =/ chat-scry (~(scry io:sr bowl) %channels (weld flag pat) scan:tlonc) + ?~ chat-scry *chat-page + =/ rev (flop chat-scry) + ?~ rev *chat-page + =/ head i.chat-scry + =/ tail i.rev + =/ newer (some (get-chat-id head)) + =/ older (some (get-chat-id tail)) + [chat-scry older newer (lent chat-scry)] +-- |