blob: 018fa271dd7183a2fb33e283746a092280154c8e (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
/- spider, w=web2-main, tw=web2-twatter, cokis=web2-cookies
/+ strandio, string, sr=sortug, *web2-twatter
=, strand=strand:spider
=, dejs-soft:format
=, strand-fail=strand-fail:libstrand:spider
=< run
|%
++ cookie-check
|= body=@t ^- (unit ?)
=/ jon=(unit json) (better-dejson:parsing:sr body)
?~ jon ~
=/ error=(unit error-res:tw) (errors:dejs:tw u.jon)
?~ error `.y
?~ u.error ~
=/ main (head u.error)
`.n
:: :- ~ !.=(code.main 32)
::
:: :- ~ %+ roll +.u.error
:: |= [[msg=@t code=@ud] acc=@t] code
++ parse-user-id
|= body=@t ^- (unit @t)
=/ jon=(unit json) (better-dejson:parsing:sr body)
?~ jon ~
`(profile:dejs:tw u.jon)
:: ++ get-body
:: |= res=client-response:iris
:: =/ m (strand cord)
:: ^- form:m
:: ?. ?=(%finished -.res) (strand-fail %no-body ~)
:: ?~ full-file.res (strand-fail %no-body ~)
:: =/ body=@t q.data.u.full-file.res
:: (pure:m body)
++ get-body
|= res=client-response:iris ^- @t
?. ?=(%finished -.res) ''
?~ full-file.res ''
q.data.u.full-file.res
++ coki-to-string
|= t=twatter-creds:cokis ^- cord
=/ at (trip auth-token.t)
=/ ct0 (trip ct0.t)
=/ kdt (trip kdt.t)
=/ twid (trip twid.t)
%- crip
"auth_token={at};ct0={ct0};kdt={kdt};twid={twid};"
++ run
^- thread:spider
|= arg=vase
=/ m (strand vase)
^- form:m
|^
=/ req !<((unit request:w) arg) :: dojo only?
?~ req (pure:m !>([%fail 'wrong request']))
?. ?=(%twatter -.u.req) (pure:m !>([%fail 'wrong request']))
=/ =action:tw +.u.req
?: ?=(%lurk -.action) (lurk thread.action)
:: Else is logged in, need cookies
;< =bowl:spider bind:m get-bowl:strandio
=/ coki (scry:io:sr %trill-cookies /app/twatter scry:cokis our.bowl now.bowl)
?: ?=(%ng -.coki) (pure:m !>([%twatter %no-coki ~]))
?: ?=(%active -.coki) (pure:m !>([%twatter %no-coki ~]))
?. ?=(%twatter -.p.coki) (pure:m !>([%twatter %no-coki ~]))
=/ csrf ct0.p.coki
=/ tw=twatter-creds:cokis +.p.coki
=/ coki-string (coki-to-string tw)
=/ headers (logged-headers coki-string csrf)
?+ -.action (pure:m !>([%fail 'wrong request']))
%user
=/ vars (build-variables ~[['screen_name' %s username.action]])
=/ url %- crip (weld (burl user-by-name:urls) "?variables={vars}&features={features}")
=/ req1 [%'GET' url headers ~]
;< ~ bind:m (send-request:strandio req1)
;< res1=client-response:iris bind:m take-client-response:strandio
=/ body1=@t (get-body res1)
=/ coki-ok (cookie-check body1)
?~ coki-ok (pure:m !>([%fail 'parsing error']))
?. u.coki-ok
(pure:m !>([%twatter %no-coki ~]))
=/ user-id (parse-user-id body1)
?~ user-id (pure:m !>([%fail 'parsing error']))
=/ url2 %- crip (weld (burl user-tweets:urls) (userid-params u.user-id))
=/ req2 [%'GET' url2 headers ~]
;< ~ bind:m (send-request:strandio req2)
;< res2=client-response:iris bind:m take-client-response:strandio
=/ body2 (get-body res2)
(pure:m !>([%twatter %user body1 body2]))
:: ?~ user-id (pure:m !>(`response:tw`[%user profile=`user-profile feed=~]))
:: =/ tw-data-req ?- replies.u.command
:: %| (twatter-request (build-request-url 'userId' u.user-id cursor.u.command user-tweets:endpoints) gt csrf)
:: %& (twatter-request (build-request-url 'userId' u.user-id cursor.u.command user-tweets-replies:endpoints) gt csrf) ==
%thread
=/ vars (build-variables ~[['focalTweetId' %s id.action]])
=/ url %- crip (weld (burl tweet-detail:urls) "?variables={vars}&features={features}")
:: =/ tw-data-req (embed-request id.u.command)
:: =/ req (twatter-request url gt csrf)
=/ req [%'GET' url headers ~]
:: =/ tw-data-req (twatter-request (build-request-v2-url 'tweetId' id.u.command cursor.u.command tweet-lurk:endpoints) gt csrf)
;< ~ bind:m (send-request:strandio req)
;< res=client-response:iris bind:m take-client-response:strandio
=/ body (get-body res)
=/ coki-ok (cookie-check body)
?~ coki-ok (pure:m !>([%fail 'parsing error']))
?. u.coki-ok
:: (lurk id.action)
(pure:m !>([%twatter %no-coki ~]))
(pure:m !>([%twatter %thread body]))
%search
=/ vars (build-variables ~[['rawQuery' %s query.action] ['querySource' %s 'typed_query'] ['product' %s 'Latest']])
=/ url %- crip (weld (burl search-timeline:urls) "?variables={vars}&features={features}")
:: =/ tw-data-req (twatter-request (build-search-url query.u.command cursor.u.command) gt csrf)
=/ req [%'GET' url headers ~]
;< ~ bind:m (send-request:strandio req)
;< res=client-response:iris bind:m take-client-response:strandio
=/ body (get-body res)
(pure:m !>([%twatter %search query.action body]))
%hark
=/ vars "?{notes-params}"
=/ url %- crip (weld notifications-url vars)
=/ req [%'GET' url headers ~]
;< ~ bind:m (send-request:strandio req)
;< res=client-response:iris bind:m take-client-response:strandio
=/ body (get-body res)
(pure:m !>([%twatter %hark body]))
:: posts
%post
=/ url (crip (burl create-tweet:urls))
=/ body=octs (build-post-body +.action)
=/ req [%'POST' url headers `body]
;< ~ bind:m (send-request:strandio req)
;< res=client-response:iris bind:m take-client-response:strandio
=/ res-body (get-body res)
:: TODO error handle. If successful the body returns a {data: {create_tweet: tweet_results...}} object
(pure:m !>([%twatter %post-ack ~]))
==
++ lurk
|= tweet-id=@t
=/ csrf-req csrf-token-request :: TODO could cache gt and crsf for a while, instead of fetching on every request
;< ~ bind:m (send-request:strandio csrf-req)
;< csrf-res=client-response:iris bind:m take-client-response:strandio
;< csrf=@t bind:m (parse-csrf csrf-res)
=/ guest-token-req gt-request
;< ~ bind:m (send-request:strandio guest-token-req)
;< gt-res=client-response:iris bind:m take-client-response:strandio
;< gt=@t bind:m (parse-gt gt-res)
:: turns out we do need this
=/ vars (build-variables ~[['tweetId' %s tweet-id]])
=/ url %- crip (weld (burl tweet-lurk:urls) "?variables={vars}&features={features}")
=/ req1 (twatter-request url gt csrf)
:: =/ req1 [%'GET' url lurk-headers ~]
;< ~ bind:m (send-request:strandio req1)
;< res1=client-response:iris bind:m take-client-response:strandio
=/ body1 (get-body res1)
(pure:m !>([%twatter %thread-lurk body1]))
--
--
|
