module Utils.HTTP ( get ) where import Control.Lens ((^.)) import qualified Control.Logging as Logging import qualified Data.ByteString.Lazy as BS import Data.Text (Text) import qualified Data.Text as T import Data.Text.Encoding as T import qualified Network.Wreq as Wreq import Network.Wreq.Session (Session) import qualified Network.Wreq.Session as Session import qualified Network.Wreq.Types as Types import Model.URL get :: Session -> URL -> IO (Either Text Text) get session url = do let options = Wreq.defaults { Types.headers = headers } response <- Session.getWith options session (T.unpack url) let body = T.decodeUtf8 . BS.toStrict $ response ^. Wreq.responseBody let statusCode = response ^. Wreq.responseStatus ^. Wreq.statusCode if statusCode >= 200 && statusCode < 300 then return . Right $ body else do Logging.loggingLogger Logging.LevelError "" . T.concat $ [ "Got status " , T.pack . show $ statusCode , " while fetching " , url , ":\n" , body ] return . Left $ body where headers = [ ("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0") , ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") , ("Accept-Language", "en-US,en;fr;q=0.5") , ("Referer", "https://duckduckgo.com/") , ("DNT", "1") , ("Connection", "keep-alive") , ("Upgrade-Insecure-Requests", "1") ]