/-  spider, w=web2-main, tw=web2-twatter, cokis=web2-cookies
/+  strandio, string, sr=sortug, *web2-twatter
=,  strand=strand:spider
=,  dejs-soft:format
=,  strand-fail=strand-fail:libstrand:spider
=<  run
|%
++  cookie-check
  |=  body=@t  ^-  (unit ?)
  =/  jon=(unit json)  (better-dejson:parsing:sr body)
  ?~  jon  ~
  =/  error=(unit error-res:tw)  (errors:dejs:tw u.jon)
  ?~  error  `.y
  ?~  u.error  ~
  =/  main  (head u.error)
  `.n
  :: :-  ~  !.=(code.main 32)
  ::
 :: :-  ~  %+  roll  +.u.error
 :: |=  [[msg=@t code=@ud] acc=@t]  code
  
++  parse-user-id
  |=  body=@t  ^-  (unit @t)
  =/  jon=(unit json)  (better-dejson:parsing:sr body)
  ?~  jon  ~
  `(profile:dejs:tw u.jon)
:: ++  get-body
:: |=  res=client-response:iris
:: =/  m  (strand cord)
:: ^-  form:m
:: ?.  ?=(%finished -.res)  (strand-fail %no-body ~)
:: ?~  full-file.res  (strand-fail %no-body ~)
:: =/  body=@t  q.data.u.full-file.res
:: (pure:m body)  
++  get-body
|=  res=client-response:iris  ^-  @t
?.  ?=(%finished -.res)  ''
?~  full-file.res  ''
q.data.u.full-file.res
++  coki-to-string
|=  t=twatter-creds:cokis  ^-  cord
=/  at  (trip auth-token.t)
=/  ct0  (trip ct0.t)
=/  kdt  (trip kdt.t)
=/  twid  (trip twid.t)
%-  crip
"auth_token={at};ct0={ct0};kdt={kdt};twid={twid};"
++  run
  ^-  thread:spider
  |=  arg=vase
  =/  m  (strand vase)
  ^-  form:m
  |^
  =/  req  !<((unit request:w) arg)  ::  dojo only?
  ?~  req   (pure:m !>([%fail 'wrong request']))
  ?.  ?=(%twatter -.u.req)  (pure:m !>([%fail 'wrong request']))
  =/  =action:tw  +.u.req
  ?:  ?=(%lurk -.action)  (lurk thread.action)
  :: Else is logged in, need cookies
  ;<  =bowl:spider  bind:m  get-bowl:strandio
  =/  coki  (scry:io:sr %trill-cookies /app/twatter scry:cokis our.bowl now.bowl)
  ?:  ?=(%ng -.coki)  (pure:m !>([%twatter %no-coki ~]))
  ?:  ?=(%active -.coki)  (pure:m !>([%twatter %no-coki ~]))
  ?.  ?=(%twatter -.p.coki)  (pure:m !>([%twatter %no-coki ~]))
  =/  csrf  ct0.p.coki
  =/  tw=twatter-creds:cokis  +.p.coki
  =/  coki-string  (coki-to-string tw)
  =/  headers  (logged-headers coki-string csrf)
  ?+  -.action  (pure:m !>([%fail 'wrong request']))
  %user
    =/  vars  (build-variables ~[['screen_name' %s username.action]])
    =/  url  %-  crip  (weld (burl user-by-name:urls) "?variables={vars}&features={features}")
    =/  req1  [%'GET' url headers ~]
    ;<  ~  bind:m  (send-request:strandio req1)
    ;<  res1=client-response:iris  bind:m  take-client-response:strandio
    =/  body1=@t  (get-body res1)
    =/  coki-ok  (cookie-check body1)
    ?~  coki-ok  (pure:m !>([%fail 'parsing error']))
    ?.  u.coki-ok  
    (pure:m !>([%twatter %no-coki ~]))
     =/  user-id  (parse-user-id body1)
     ?~  user-id  (pure:m !>([%fail 'parsing error']))
     =/  url2  %-  crip  (weld (burl user-tweets:urls) (userid-params u.user-id))
     =/  req2  [%'GET' url2 headers ~]
     ;<  ~  bind:m  (send-request:strandio req2)
     ;<  res2=client-response:iris  bind:m  take-client-response:strandio
     =/  body2  (get-body res2)
     (pure:m !>([%twatter %user body1 body2]))
  ::   ?~  user-id  (pure:m !>(`response:tw`[%user profile=`user-profile feed=~]))
  ::   =/  tw-data-req  ?-  replies.u.command
  ::     %|  (twatter-request (build-request-url 'userId' u.user-id cursor.u.command user-tweets:endpoints) gt csrf)
  ::     %&  (twatter-request (build-request-url 'userId' u.user-id cursor.u.command user-tweets-replies:endpoints) gt csrf)  ==
  %thread
    =/  vars  (build-variables ~[['focalTweetId' %s id.action]])
    =/  url  %-  crip  (weld (burl tweet-detail:urls) "?variables={vars}&features={features}")
    :: =/  tw-data-req     (embed-request id.u.command)
    :: =/  req  (twatter-request url gt csrf)
    =/  req  [%'GET' url headers ~]
    ::  =/  tw-data-req  (twatter-request (build-request-v2-url 'tweetId' id.u.command cursor.u.command tweet-lurk:endpoints) gt csrf)
    ;<  ~  bind:m  (send-request:strandio req)
    ;<  res=client-response:iris  bind:m  take-client-response:strandio
    =/  body  (get-body res)
    =/  coki-ok  (cookie-check body)
    ?~  coki-ok  (pure:m !>([%fail 'parsing error']))
    ?.  u.coki-ok  
    :: (lurk id.action)
    (pure:m !>([%twatter %no-coki ~]))
    (pure:m !>([%twatter %thread body]))
  %search
    =/  vars  (build-variables ~[['rawQuery' %s query.action] ['querySource' %s 'typed_query'] ['product' %s 'Latest']])
    =/  url  %-  crip  (weld (burl search-timeline:urls) "?variables={vars}&features={features}")
    :: =/  tw-data-req  (twatter-request (build-search-url query.u.command cursor.u.command) gt csrf)
    =/  req  [%'GET' url headers ~]
    ;<  ~  bind:m  (send-request:strandio req)
    ;<  res=client-response:iris  bind:m  take-client-response:strandio
    =/  body  (get-body res)
    (pure:m !>([%twatter %search query.action body]))
  %hark
    =/  vars  "?{notes-params}"
    =/  url  %-  crip  (weld notifications-url vars)
    =/  req  [%'GET' url headers ~]
    ;<  ~  bind:m  (send-request:strandio req)
    ;<  res=client-response:iris  bind:m  take-client-response:strandio
    =/  body  (get-body res)
    (pure:m !>([%twatter %hark body]))
  ::  posts
  %post
    =/  url  (crip (burl create-tweet:urls))
    =/  body=octs  (build-post-body +.action)
    =/  req  [%'POST' url headers `body]
    ;<  ~  bind:m  (send-request:strandio req)
    ;<  res=client-response:iris  bind:m  take-client-response:strandio
    =/  res-body  (get-body res)
    :: TODO error handle. If successful the body returns a {data: {create_tweet: tweet_results...}} object
    (pure:m !>([%twatter %post-ack ~]))
  ==
  ++  lurk
    |=  tweet-id=@t
    =/  csrf-req   csrf-token-request  :: TODO could cache gt and crsf for a while, instead of fetching on every request
    ;<  ~  bind:m  (send-request:strandio csrf-req)
    ;<  csrf-res=client-response:iris  bind:m  take-client-response:strandio
    ;<  csrf=@t  bind:m  (parse-csrf csrf-res)
    =/  guest-token-req  gt-request
    ;<  ~  bind:m  (send-request:strandio guest-token-req)
    ;<  gt-res=client-response:iris  bind:m  take-client-response:strandio
    ;<  gt=@t  bind:m  (parse-gt gt-res)
    :: turns out we do need this
    =/  vars  (build-variables ~[['tweetId' %s tweet-id]])
    =/  url  %-  crip  (weld (burl tweet-lurk:urls) "?variables={vars}&features={features}")
    =/  req1  (twatter-request url gt csrf)
    :: =/  req1  [%'GET' url lurk-headers ~]
    ;<  ~  bind:m  (send-request:strandio req1)
    ;<  res1=client-response:iris  bind:m  take-client-response:strandio
    =/  body1  (get-body res1)
    (pure:m !>([%twatter %thread-lurk body1]))

  --
--