1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
module Utils.HTTP
( get
) where
import Control.Lens ((^.))
import qualified Control.Logging as Logging
import qualified Data.ByteString.Lazy as BS
import Data.Text (Text)
import qualified Data.Text as T
import Data.Text.Encoding as T
import qualified Network.Wreq as Wreq
import Network.Wreq.Session (Session)
import qualified Network.Wreq.Session as Session
import qualified Network.Wreq.Types as Types
import Model.URL
get :: Session -> URL -> IO (Either Text Text)
get session url = do
let options = Wreq.defaults { Types.headers = headers }
response <- Session.getWith options session (T.unpack url)
let body = T.decodeUtf8 . BS.toStrict $ response ^. Wreq.responseBody
let statusCode = response ^. Wreq.responseStatus ^. Wreq.statusCode
if statusCode >= 200 && statusCode < 300 then
return . Right $ body
else do
Logging.loggingLogger Logging.LevelError "" . T.concat $
[ "Got status "
, T.pack . show $ statusCode
, " while fetching "
, url
, ":\n"
, body
]
return . Left $ body
where
headers =
[ ("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0")
, ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
, ("Accept-Language", "en-US,en;fr;q=0.5")
, ("Referer", "https://duckduckgo.com/")
, ("DNT", "1")
, ("Connection", "keep-alive")
, ("Upgrade-Insecure-Requests", "1")
]
|
