diff options
Diffstat (limited to 'desk/ted/twatter.hoon')
| -rw-r--r-- | desk/ted/twatter.hoon | 158 |
1 files changed, 158 insertions, 0 deletions
diff --git a/desk/ted/twatter.hoon b/desk/ted/twatter.hoon new file mode 100644 index 0000000..018fa27 --- /dev/null +++ b/desk/ted/twatter.hoon @@ -0,0 +1,158 @@ +/- spider, w=web2-main, tw=web2-twatter, cokis=web2-cookies +/+ strandio, string, sr=sortug, *web2-twatter +=, strand=strand:spider +=, dejs-soft:format +=, strand-fail=strand-fail:libstrand:spider +=< run +|% +++ cookie-check + |= body=@t ^- (unit ?) + =/ jon=(unit json) (better-dejson:parsing:sr body) + ?~ jon ~ + =/ error=(unit error-res:tw) (errors:dejs:tw u.jon) + ?~ error `.y + ?~ u.error ~ + =/ main (head u.error) + `.n + :: :- ~ !.=(code.main 32) + :: + :: :- ~ %+ roll +.u.error + :: |= [[msg=@t code=@ud] acc=@t] code + +++ parse-user-id + |= body=@t ^- (unit @t) + =/ jon=(unit json) (better-dejson:parsing:sr body) + ?~ jon ~ + `(profile:dejs:tw u.jon) +:: ++ get-body +:: |= res=client-response:iris +:: =/ m (strand cord) +:: ^- form:m +:: ?. ?=(%finished -.res) (strand-fail %no-body ~) +:: ?~ full-file.res (strand-fail %no-body ~) +:: =/ body=@t q.data.u.full-file.res +:: (pure:m body) +++ get-body +|= res=client-response:iris ^- @t +?. ?=(%finished -.res) '' +?~ full-file.res '' +q.data.u.full-file.res +++ coki-to-string +|= t=twatter-creds:cokis ^- cord +=/ at (trip auth-token.t) +=/ ct0 (trip ct0.t) +=/ kdt (trip kdt.t) +=/ twid (trip twid.t) +%- crip +"auth_token={at};ct0={ct0};kdt={kdt};twid={twid};" +++ run + ^- thread:spider + |= arg=vase + =/ m (strand vase) + ^- form:m + |^ + =/ req !<((unit request:w) arg) :: dojo only? + ?~ req (pure:m !>([%fail 'wrong request'])) + ?. ?=(%twatter -.u.req) (pure:m !>([%fail 'wrong request'])) + =/ =action:tw +.u.req + ?: ?=(%lurk -.action) (lurk thread.action) + :: Else is logged in, need cookies + ;< =bowl:spider bind:m get-bowl:strandio + =/ coki (scry:io:sr %trill-cookies /app/twatter scry:cokis our.bowl now.bowl) + ?: ?=(%ng -.coki) (pure:m !>([%twatter %no-coki ~])) + ?: ?=(%active -.coki) (pure:m !>([%twatter %no-coki ~])) + ?. ?=(%twatter -.p.coki) (pure:m !>([%twatter %no-coki ~])) + =/ csrf ct0.p.coki + =/ tw=twatter-creds:cokis +.p.coki + =/ coki-string (coki-to-string tw) + =/ headers (logged-headers coki-string csrf) + ?+ -.action (pure:m !>([%fail 'wrong request'])) + %user + =/ vars (build-variables ~[['screen_name' %s username.action]]) + =/ url %- crip (weld (burl user-by-name:urls) "?variables={vars}&features={features}") + =/ req1 [%'GET' url headers ~] + ;< ~ bind:m (send-request:strandio req1) + ;< res1=client-response:iris bind:m take-client-response:strandio + =/ body1=@t (get-body res1) + =/ coki-ok (cookie-check body1) + ?~ coki-ok (pure:m !>([%fail 'parsing error'])) + ?. u.coki-ok + (pure:m !>([%twatter %no-coki ~])) + =/ user-id (parse-user-id body1) + ?~ user-id (pure:m !>([%fail 'parsing error'])) + =/ url2 %- crip (weld (burl user-tweets:urls) (userid-params u.user-id)) + =/ req2 [%'GET' url2 headers ~] + ;< ~ bind:m (send-request:strandio req2) + ;< res2=client-response:iris bind:m take-client-response:strandio + =/ body2 (get-body res2) + (pure:m !>([%twatter %user body1 body2])) + :: ?~ user-id (pure:m !>(`response:tw`[%user profile=`user-profile feed=~])) + :: =/ tw-data-req ?- replies.u.command + :: %| (twatter-request (build-request-url 'userId' u.user-id cursor.u.command user-tweets:endpoints) gt csrf) + :: %& (twatter-request (build-request-url 'userId' u.user-id cursor.u.command user-tweets-replies:endpoints) gt csrf) == + %thread + =/ vars (build-variables ~[['focalTweetId' %s id.action]]) + =/ url %- crip (weld (burl tweet-detail:urls) "?variables={vars}&features={features}") + :: =/ tw-data-req (embed-request id.u.command) + :: =/ req (twatter-request url gt csrf) + =/ req [%'GET' url headers ~] + :: =/ tw-data-req (twatter-request (build-request-v2-url 'tweetId' id.u.command cursor.u.command tweet-lurk:endpoints) gt csrf) + ;< ~ bind:m (send-request:strandio req) + ;< res=client-response:iris bind:m take-client-response:strandio + =/ body (get-body res) + =/ coki-ok (cookie-check body) + ?~ coki-ok (pure:m !>([%fail 'parsing error'])) + ?. u.coki-ok + :: (lurk id.action) + (pure:m !>([%twatter %no-coki ~])) + (pure:m !>([%twatter %thread body])) + %search + =/ vars (build-variables ~[['rawQuery' %s query.action] ['querySource' %s 'typed_query'] ['product' %s 'Latest']]) + =/ url %- crip (weld (burl search-timeline:urls) "?variables={vars}&features={features}") + :: =/ tw-data-req (twatter-request (build-search-url query.u.command cursor.u.command) gt csrf) + =/ req [%'GET' url headers ~] + ;< ~ bind:m (send-request:strandio req) + ;< res=client-response:iris bind:m take-client-response:strandio + =/ body (get-body res) + (pure:m !>([%twatter %search query.action body])) + %hark + =/ vars "?{notes-params}" + =/ url %- crip (weld notifications-url vars) + =/ req [%'GET' url headers ~] + ;< ~ bind:m (send-request:strandio req) + ;< res=client-response:iris bind:m take-client-response:strandio + =/ body (get-body res) + (pure:m !>([%twatter %hark body])) + :: posts + %post + =/ url (crip (burl create-tweet:urls)) + =/ body=octs (build-post-body +.action) + =/ req [%'POST' url headers `body] + ;< ~ bind:m (send-request:strandio req) + ;< res=client-response:iris bind:m take-client-response:strandio + =/ res-body (get-body res) + :: TODO error handle. If successful the body returns a {data: {create_tweet: tweet_results...}} object + (pure:m !>([%twatter %post-ack ~])) + == + ++ lurk + |= tweet-id=@t + =/ csrf-req csrf-token-request :: TODO could cache gt and crsf for a while, instead of fetching on every request + ;< ~ bind:m (send-request:strandio csrf-req) + ;< csrf-res=client-response:iris bind:m take-client-response:strandio + ;< csrf=@t bind:m (parse-csrf csrf-res) + =/ guest-token-req gt-request + ;< ~ bind:m (send-request:strandio guest-token-req) + ;< gt-res=client-response:iris bind:m take-client-response:strandio + ;< gt=@t bind:m (parse-gt gt-res) + :: turns out we do need this + =/ vars (build-variables ~[['tweetId' %s tweet-id]]) + =/ url %- crip (weld (burl tweet-lurk:urls) "?variables={vars}&features={features}") + =/ req1 (twatter-request url gt csrf) + :: =/ req1 [%'GET' url lurk-headers ~] + ;< ~ bind:m (send-request:strandio req1) + ;< res1=client-response:iris bind:m take-client-response:strandio + =/ body1 (get-body res1) + (pure:m !>([%twatter %thread-lurk body1])) + + -- +-- |