From 149a0470b73781022e584aaeaa7ce871d6f4173b Mon Sep 17 00:00:00 2001 From: Joris Date: Tue, 19 Jun 2018 22:49:16 +0200 Subject: Add automatic tests on remote pages --- ad-listener.cabal | 15 ++-- src/executable/haskell/Service/AdListener.hs | 59 ++++--------- src/executable/haskell/Utils/Either.hs | 7 -- src/executable/haskell/Utils/HTTP.hs | 19 ---- src/lib/haskell/FetchAd.hs | 36 ++++++++ src/lib/haskell/Model/Ad.hs | 22 +++++ src/lib/haskell/Model/URL.hs | 7 ++ src/lib/haskell/Parser/LeboncoinParser.hs | 24 +++++ src/lib/haskell/Parser/OuestFranceParser.hs | 25 ++++++ src/lib/haskell/Parser/SeLogerParser.hs | 24 +++++ src/lib/haskell/Parser/Utils.hs | 48 ++++++++++ src/lib/haskell/Utils/HTTP.hs | 22 +++++ src/parser/haskell/Model/Ad.hs | 22 ----- src/parser/haskell/Model/URL.hs | 7 -- src/parser/haskell/Parser/LeboncoinParser.hs | 24 ----- src/parser/haskell/Parser/OuestFranceParser.hs | 25 ------ src/parser/haskell/Parser/SeLogerParser.hs | 24 ----- src/parser/haskell/Parser/Utils.hs | 48 ---------- src/test/haskell/Ads.hs | 97 +++++++++++++++++++++ src/test/haskell/Main.hs | 116 +++++-------------------- 20 files changed, 353 insertions(+), 318 deletions(-) delete mode 100644 src/executable/haskell/Utils/Either.hs delete mode 100644 src/executable/haskell/Utils/HTTP.hs create mode 100644 src/lib/haskell/FetchAd.hs create mode 100644 src/lib/haskell/Model/Ad.hs create mode 100644 src/lib/haskell/Model/URL.hs create mode 100644 src/lib/haskell/Parser/LeboncoinParser.hs create mode 100644 src/lib/haskell/Parser/OuestFranceParser.hs create mode 100644 src/lib/haskell/Parser/SeLogerParser.hs create mode 100644 src/lib/haskell/Parser/Utils.hs create mode 100644 src/lib/haskell/Utils/HTTP.hs delete mode 100644 src/parser/haskell/Model/Ad.hs delete mode 100644 src/parser/haskell/Model/URL.hs delete mode 100644 src/parser/haskell/Parser/LeboncoinParser.hs delete mode 100644 src/parser/haskell/Parser/OuestFranceParser.hs delete mode 100644 src/parser/haskell/Parser/SeLogerParser.hs delete mode 100644 src/parser/haskell/Parser/Utils.hs create mode 100644 src/test/haskell/Ads.hs diff --git a/ad-listener.cabal b/ad-listener.cabal index b3598c2..793fd5a 100644 --- a/ad-listener.cabal +++ b/ad-listener.cabal @@ -8,7 +8,7 @@ Build-type: Simple Cabal-version: >= 1.10 Library - Hs-source-dirs: src/parser/haskell + Hs-source-dirs: src/lib/haskell Main-is: Main.hs Ghc-options: -Wall -Werror Default-language: Haskell2010 @@ -18,15 +18,19 @@ Library Build-depends: base + , bytestring + , http-conduit , tagsoup , text Exposed-modules: - Model.Ad + FetchAd + , Model.Ad , Model.URL , Parser.LeboncoinParser , Parser.OuestFranceParser , Parser.SeLogerParser + , Utils.HTTP Other-modules: Parser.Utils @@ -45,12 +49,10 @@ Executable ad-listener , ad-listener , blaze-html , blaze-markup - , bytestring , clay , config-manager , containers , directory - , http-conduit , mime-mail , tagsoup , text @@ -61,8 +63,6 @@ Executable ad-listener , Model.Mail , Service.AdListener , Service.MailService - , Utils.Either - , Utils.HTTP , Utils.Time , View.Ad @@ -81,3 +81,6 @@ Test-suite test , hspec , ad-listener , text + + Other-modules: + Ads diff --git a/src/executable/haskell/Service/AdListener.hs b/src/executable/haskell/Service/AdListener.hs index f0adbb8..bbd06d9 100644 --- a/src/executable/haskell/Service/AdListener.hs +++ b/src/executable/haskell/Service/AdListener.hs @@ -2,25 +2,20 @@ module Service.AdListener ( start ) where -import Control.Concurrent (threadDelay) -import Data.Either (rights) -import Data.Text.Encoding as T -import qualified Data.Text.IO as T -import Prelude hiding (error) +import Control.Concurrent (threadDelay) +import qualified Data.Text.IO as T +import Prelude hiding (error) -import Conf (Conf) +import Conf (Conf) import qualified Conf -import Model.Ad (Ad) -import qualified Model.Ad as Ad -import Model.Mail (Mail (Mail)) -import Model.URL (URL) -import qualified Parser.LeboncoinParser as LeboncoinParser -import qualified Parser.OuestFranceParser as OuestFranceParser -import qualified Parser.SeLogerParser as SeLogerParser -import qualified Service.MailService as MailService -import qualified Utils.HTTP as HTTP -import qualified Utils.Time as TimeUtils -import qualified View.Ad as Ad +import qualified FetchAd +import Model.Ad (Ad) +import qualified Model.Ad as Ad +import Model.Mail (Mail (Mail)) +import Model.URL (URL) +import qualified Service.MailService as MailService +import qualified Utils.Time as TimeUtils +import qualified View.Ad as Ad start :: Conf -> IO () start conf = do @@ -49,39 +44,15 @@ listenToNewAdsWithViewedURLs conf viewedURLs = do fetchAds :: Conf -> IO [Ad] fetchAds conf = do - leboncoinAds <- getLeboncoinAds conf - ouestFranceAds <- getOuestFranceAds conf - seLogerAds <- getSeLogerAds conf + leboncoinAds <- FetchAd.leboncoin (Conf.leboncoinUrls conf) + ouestFranceAds <- FetchAd.ouestFrance (Conf.ouestFranceUrls conf) + seLogerAds <- FetchAd.seLoger (Conf.seLogerUrls conf) let results = leboncoinAds ++ ouestFranceAds ++ seLogerAds if null results then T.putStrLn "Parsed 0 results!" else return () return results -getLeboncoinAds :: Conf -> IO [Ad] -getLeboncoinAds conf = - fmap (concat . map LeboncoinParser.parse . rights) - . sequence - . map (HTTP.get T.decodeLatin1) - . Conf.leboncoinUrls - $ conf - -getOuestFranceAds :: Conf -> IO [Ad] -getOuestFranceAds conf = - fmap (concat . map OuestFranceParser.parse . rights) - . sequence - . map (HTTP.get T.decodeUtf8) - . Conf.ouestFranceUrls - $ conf - -getSeLogerAds :: Conf -> IO [Ad] -getSeLogerAds conf = - fmap (concat . map SeLogerParser.parse . rights) - . sequence - . map (HTTP.get T.decodeUtf8) - . Conf.seLogerUrls - $ conf - sendMail :: Conf -> [Ad] -> IO () sendMail conf ads = let (title, plainBody) = Ad.renderAds ads diff --git a/src/executable/haskell/Utils/Either.hs b/src/executable/haskell/Utils/Either.hs deleted file mode 100644 index 5d62dcc..0000000 --- a/src/executable/haskell/Utils/Either.hs +++ /dev/null @@ -1,7 +0,0 @@ -module Utils.Either - ( mapLeft - ) where - -mapLeft :: (a -> c) -> Either a b -> Either c b -mapLeft f (Left l) = Left (f l) -mapLeft _ (Right r) = (Right r) diff --git a/src/executable/haskell/Utils/HTTP.hs b/src/executable/haskell/Utils/HTTP.hs deleted file mode 100644 index 919e66d..0000000 --- a/src/executable/haskell/Utils/HTTP.hs +++ /dev/null @@ -1,19 +0,0 @@ -module Utils.HTTP - ( get - ) where - -import Control.Exception (SomeException, try) -import Data.ByteString (ByteString) -import qualified Data.ByteString.Lazy as BS -import Data.Text (Text) -import qualified Data.Text as T -import Network.HTTP.Conduit - -import Model.URL -import Utils.Either (mapLeft) - -get :: (ByteString -> Text) -> URL -> IO (Either Text Text) -get decode url = mapLeft (T.pack . show) <$> (try (unsafeGetPage decode url) :: IO (Either SomeException Text)) - -unsafeGetPage :: (ByteString -> Text) -> URL -> IO Text -unsafeGetPage decode url = (decode . BS.toStrict) <$> simpleHttp (T.unpack url) diff --git a/src/lib/haskell/FetchAd.hs b/src/lib/haskell/FetchAd.hs new file mode 100644 index 0000000..a206181 --- /dev/null +++ b/src/lib/haskell/FetchAd.hs @@ -0,0 +1,36 @@ +module FetchAd + ( leboncoin + , ouestFrance + , seLoger + ) where + +import Data.Either (rights) +import Data.Text.Encoding as T + +import Model.Ad (Ad) +import Model.URL (URL) +import qualified Parser.LeboncoinParser as LeboncoinParser +import qualified Parser.OuestFranceParser as OuestFranceParser +import qualified Parser.SeLogerParser as SeLogerParser +import qualified Utils.HTTP as HTTP + +leboncoin :: [URL] -> IO [Ad] +leboncoin urls = + fmap (concat . map LeboncoinParser.parse . rights) + . sequence + . map (HTTP.get T.decodeLatin1) + $ urls + +ouestFrance :: [URL] -> IO [Ad] +ouestFrance urls = + fmap (concat . map OuestFranceParser.parse . rights) + . sequence + . map (HTTP.get T.decodeUtf8) + $ urls + +seLoger :: [URL] -> IO [Ad] +seLoger urls = + fmap (concat . map SeLogerParser.parse . rights) + . sequence + . map (HTTP.get T.decodeUtf8) + $ urls diff --git a/src/lib/haskell/Model/Ad.hs b/src/lib/haskell/Model/Ad.hs new file mode 100644 index 0000000..06906eb --- /dev/null +++ b/src/lib/haskell/Model/Ad.hs @@ -0,0 +1,22 @@ +module Model.Ad + ( Ad(..) + , getNewAds + ) where + +import Data.List ((\\)) +import Data.Text (Text) + +import Model.URL (URL) + +data Ad = Ad + { name :: Text + , location :: Text + , price :: Maybe Text + , url :: URL + } deriving (Eq, Read, Show) + +getNewAds :: [URL] -> [Ad] -> ([URL], [Ad]) +getNewAds viewdURLs ads = + let newURLs = (map url ads) \\ viewdURLs + newAds = filter (\ad -> elem (url ad) newURLs) ads + in (newURLs, newAds) diff --git a/src/lib/haskell/Model/URL.hs b/src/lib/haskell/Model/URL.hs new file mode 100644 index 0000000..2114113 --- /dev/null +++ b/src/lib/haskell/Model/URL.hs @@ -0,0 +1,7 @@ +module Model.URL + ( URL + ) where + +import Data.Text + +type URL = Text diff --git a/src/lib/haskell/Parser/LeboncoinParser.hs b/src/lib/haskell/Parser/LeboncoinParser.hs new file mode 100644 index 0000000..77213cb --- /dev/null +++ b/src/lib/haskell/Parser/LeboncoinParser.hs @@ -0,0 +1,24 @@ +module Parser.LeboncoinParser + ( parse + ) where + +import Data.Maybe (catMaybes) +import Data.Text (Text) +import qualified Data.Text as T +import Text.HTML.TagSoup + +import Model.Ad (Ad (Ad)) +import Parser.Utils + +parse :: Text -> [Ad] +parse page = + catMaybes . fmap parseAd $ partitions (~== (T.unpack "")) tags + where tags = getTagsBetween "
  • " "
    " (parseTags page) + +parseAd :: [Tag Text] -> Maybe Ad +parseAd tags = do + name <- getTagTextAfter "

    " tags + location <- getTagAttribute "" "content" tags + let price = getTagTextAfter "

    " tags + url <- getTagAttribute "" "href" tags + return (Ad name location price (T.concat ["https:", url])) diff --git a/src/lib/haskell/Parser/OuestFranceParser.hs b/src/lib/haskell/Parser/OuestFranceParser.hs new file mode 100644 index 0000000..f46ed03 --- /dev/null +++ b/src/lib/haskell/Parser/OuestFranceParser.hs @@ -0,0 +1,25 @@ +module Parser.OuestFranceParser + ( parse + ) where + +import Data.Maybe (catMaybes) +import Data.Text (Text) +import qualified Data.Text as T +import Text.HTML.TagSoup + +import Model.Ad (Ad (Ad)) +import Parser.Utils + +parse :: Text -> [Ad] +parse page = + catMaybes . fmap parseAd $ partitions (~== (T.unpack "")) tags + where tags = getTagsBetween "
    " "
    " (parseTags page) + +parseAd :: [Tag Text] -> Maybe Ad +parseAd tags = do + name <- getTagTextAfter "" tags + location <- getTagTextAfter "" tags + let price = getTagTextAfter "" tags + let startUrl = "https://www.ouestfrance-immo.com/" + url <- getTagAttribute "" "href" tags + return (Ad name location price (T.concat [startUrl, url])) diff --git a/src/lib/haskell/Parser/SeLogerParser.hs b/src/lib/haskell/Parser/SeLogerParser.hs new file mode 100644 index 0000000..b073862 --- /dev/null +++ b/src/lib/haskell/Parser/SeLogerParser.hs @@ -0,0 +1,24 @@ +module Parser.SeLogerParser + ( parse + ) where + +import Data.Maybe (catMaybes) +import Data.Text (Text) +import qualified Data.Text as T +import Text.HTML.TagSoup + +import Model.Ad (Ad (Ad)) +import Parser.Utils + +parse :: Text -> [Ad] +parse page = + catMaybes . fmap parseAd $ partitions (~== (T.unpack "
    ")) tags + where tags = getTagsBetween "
    " "
    " (parseTags page) + +parseAd :: [Tag Text] -> Maybe Ad +parseAd tags = do + name <- getTagTextAfter "" tags + location <- getTagTextAfter "
    " tags + let price = getTagTextAfter "" tags + url <- getTagAttribute "" "href" tags + return (Ad name location price url) diff --git a/src/lib/haskell/Parser/Utils.hs b/src/lib/haskell/Parser/Utils.hs new file mode 100644 index 0000000..4768327 --- /dev/null +++ b/src/lib/haskell/Parser/Utils.hs @@ -0,0 +1,48 @@ +module Parser.Utils + ( getTagsBefore + , getTagsAfter + , getTagsBetween + , getTagAttributes + , getTagAttribute + , getTagTextAfter + ) where + +import Data.List (find, findIndex) +import Data.Maybe (catMaybes, listToMaybe) +import Data.Text (Text) +import qualified Data.Text as T +import Text.HTML.TagSoup + +getTagsBefore :: Text -> [Tag Text] -> [Tag Text] +getTagsBefore selector = takeWhile (~/= (T.unpack selector)) + +getTagsAfter :: Text -> [Tag Text] -> [Tag Text] +getTagsAfter selector = drop 1 . dropWhile (~/= (T.unpack selector)) + +getTagsBetween :: Text -> Text -> [Tag Text] -> [Tag Text] +getTagsBetween begin end = getTagsBefore end . getTagsAfter begin + +getTagAttributes :: Text -> Text -> [Tag Text] -> [Text] +getTagAttributes selector attribute = + catMaybes + . fmap (maybeTagAttribute attribute) + . filter (~== (T.unpack selector)) + +getTagAttribute :: Text -> Text -> [Tag Text] -> Maybe Text +getTagAttribute selector attribute = + listToMaybe + . getTagAttributes selector attribute + +getTagTextAfter :: Text -> [Tag Text] -> Maybe Text +getTagTextAfter selector tags = + case findIndex (~== (T.unpack selector)) tags of + Just index -> fmap T.strip $ safeGetAt (index + 1) tags >>= maybeTagText + Nothing -> Nothing + +maybeTagAttribute :: Text -> Tag Text -> Maybe Text +maybeTagAttribute name (TagOpen _ xs) = + fmap snd . find (\(x, _) -> x == name) $ xs +maybeTagAttribute _ _ = Nothing + +safeGetAt :: Int -> [a] -> Maybe a +safeGetAt index = listToMaybe . drop index diff --git a/src/lib/haskell/Utils/HTTP.hs b/src/lib/haskell/Utils/HTTP.hs new file mode 100644 index 0000000..87635ce --- /dev/null +++ b/src/lib/haskell/Utils/HTTP.hs @@ -0,0 +1,22 @@ +module Utils.HTTP + ( get + ) where + +import Control.Exception (SomeException, try) +import Data.ByteString (ByteString) +import qualified Data.ByteString.Lazy as BS +import Data.Text (Text) +import qualified Data.Text as T +import Network.HTTP.Conduit + +import Model.URL + +get :: (ByteString -> Text) -> URL -> IO (Either Text Text) +get decode url = mapLeft (T.pack . show) <$> (try (unsafeGetPage decode url) :: IO (Either SomeException Text)) + +unsafeGetPage :: (ByteString -> Text) -> URL -> IO Text +unsafeGetPage decode url = (decode . BS.toStrict) <$> simpleHttp (T.unpack url) + +mapLeft :: (a -> c) -> Either a b -> Either c b +mapLeft f (Left l) = Left (f l) +mapLeft _ (Right r) = (Right r) diff --git a/src/parser/haskell/Model/Ad.hs b/src/parser/haskell/Model/Ad.hs deleted file mode 100644 index 06906eb..0000000 --- a/src/parser/haskell/Model/Ad.hs +++ /dev/null @@ -1,22 +0,0 @@ -module Model.Ad - ( Ad(..) - , getNewAds - ) where - -import Data.List ((\\)) -import Data.Text (Text) - -import Model.URL (URL) - -data Ad = Ad - { name :: Text - , location :: Text - , price :: Maybe Text - , url :: URL - } deriving (Eq, Read, Show) - -getNewAds :: [URL] -> [Ad] -> ([URL], [Ad]) -getNewAds viewdURLs ads = - let newURLs = (map url ads) \\ viewdURLs - newAds = filter (\ad -> elem (url ad) newURLs) ads - in (newURLs, newAds) diff --git a/src/parser/haskell/Model/URL.hs b/src/parser/haskell/Model/URL.hs deleted file mode 100644 index 2114113..0000000 --- a/src/parser/haskell/Model/URL.hs +++ /dev/null @@ -1,7 +0,0 @@ -module Model.URL - ( URL - ) where - -import Data.Text - -type URL = Text diff --git a/src/parser/haskell/Parser/LeboncoinParser.hs b/src/parser/haskell/Parser/LeboncoinParser.hs deleted file mode 100644 index 77213cb..0000000 --- a/src/parser/haskell/Parser/LeboncoinParser.hs +++ /dev/null @@ -1,24 +0,0 @@ -module Parser.LeboncoinParser - ( parse - ) where - -import Data.Maybe (catMaybes) -import Data.Text (Text) -import qualified Data.Text as T -import Text.HTML.TagSoup - -import Model.Ad (Ad (Ad)) -import Parser.Utils - -parse :: Text -> [Ad] -parse page = - catMaybes . fmap parseAd $ partitions (~== (T.unpack "")) tags - where tags = getTagsBetween "
  • " "
    " (parseTags page) - -parseAd :: [Tag Text] -> Maybe Ad -parseAd tags = do - name <- getTagTextAfter "

    " tags - location <- getTagAttribute "" "content" tags - let price = getTagTextAfter "

    " tags - url <- getTagAttribute "" "href" tags - return (Ad name location price (T.concat ["https:", url])) diff --git a/src/parser/haskell/Parser/OuestFranceParser.hs b/src/parser/haskell/Parser/OuestFranceParser.hs deleted file mode 100644 index f46ed03..0000000 --- a/src/parser/haskell/Parser/OuestFranceParser.hs +++ /dev/null @@ -1,25 +0,0 @@ -module Parser.OuestFranceParser - ( parse - ) where - -import Data.Maybe (catMaybes) -import Data.Text (Text) -import qualified Data.Text as T -import Text.HTML.TagSoup - -import Model.Ad (Ad (Ad)) -import Parser.Utils - -parse :: Text -> [Ad] -parse page = - catMaybes . fmap parseAd $ partitions (~== (T.unpack "")) tags - where tags = getTagsBetween "
    " "
    " (parseTags page) - -parseAd :: [Tag Text] -> Maybe Ad -parseAd tags = do - name <- getTagTextAfter "" tags - location <- getTagTextAfter "" tags - let price = getTagTextAfter "" tags - let startUrl = "https://www.ouestfrance-immo.com/" - url <- getTagAttribute "" "href" tags - return (Ad name location price (T.concat [startUrl, url])) diff --git a/src/parser/haskell/Parser/SeLogerParser.hs b/src/parser/haskell/Parser/SeLogerParser.hs deleted file mode 100644 index b073862..0000000 --- a/src/parser/haskell/Parser/SeLogerParser.hs +++ /dev/null @@ -1,24 +0,0 @@ -module Parser.SeLogerParser - ( parse - ) where - -import Data.Maybe (catMaybes) -import Data.Text (Text) -import qualified Data.Text as T -import Text.HTML.TagSoup - -import Model.Ad (Ad (Ad)) -import Parser.Utils - -parse :: Text -> [Ad] -parse page = - catMaybes . fmap parseAd $ partitions (~== (T.unpack "
    ")) tags - where tags = getTagsBetween "
    " "
    " (parseTags page) - -parseAd :: [Tag Text] -> Maybe Ad -parseAd tags = do - name <- getTagTextAfter "" tags - location <- getTagTextAfter "
    " tags - let price = getTagTextAfter "" tags - url <- getTagAttribute "" "href" tags - return (Ad name location price url) diff --git a/src/parser/haskell/Parser/Utils.hs b/src/parser/haskell/Parser/Utils.hs deleted file mode 100644 index 4768327..0000000 --- a/src/parser/haskell/Parser/Utils.hs +++ /dev/null @@ -1,48 +0,0 @@ -module Parser.Utils - ( getTagsBefore - , getTagsAfter - , getTagsBetween - , getTagAttributes - , getTagAttribute - , getTagTextAfter - ) where - -import Data.List (find, findIndex) -import Data.Maybe (catMaybes, listToMaybe) -import Data.Text (Text) -import qualified Data.Text as T -import Text.HTML.TagSoup - -getTagsBefore :: Text -> [Tag Text] -> [Tag Text] -getTagsBefore selector = takeWhile (~/= (T.unpack selector)) - -getTagsAfter :: Text -> [Tag Text] -> [Tag Text] -getTagsAfter selector = drop 1 . dropWhile (~/= (T.unpack selector)) - -getTagsBetween :: Text -> Text -> [Tag Text] -> [Tag Text] -getTagsBetween begin end = getTagsBefore end . getTagsAfter begin - -getTagAttributes :: Text -> Text -> [Tag Text] -> [Text] -getTagAttributes selector attribute = - catMaybes - . fmap (maybeTagAttribute attribute) - . filter (~== (T.unpack selector)) - -getTagAttribute :: Text -> Text -> [Tag Text] -> Maybe Text -getTagAttribute selector attribute = - listToMaybe - . getTagAttributes selector attribute - -getTagTextAfter :: Text -> [Tag Text] -> Maybe Text -getTagTextAfter selector tags = - case findIndex (~== (T.unpack selector)) tags of - Just index -> fmap T.strip $ safeGetAt (index + 1) tags >>= maybeTagText - Nothing -> Nothing - -maybeTagAttribute :: Text -> Tag Text -> Maybe Text -maybeTagAttribute name (TagOpen _ xs) = - fmap snd . find (\(x, _) -> x == name) $ xs -maybeTagAttribute _ _ = Nothing - -safeGetAt :: Int -> [a] -> Maybe a -safeGetAt index = listToMaybe . drop index diff --git a/src/test/haskell/Ads.hs b/src/test/haskell/Ads.hs new file mode 100644 index 0000000..85a5471 --- /dev/null +++ b/src/test/haskell/Ads.hs @@ -0,0 +1,97 @@ +module Ads + ( leboncoin + , ouestFrance + , seLoger + ) where + +import Model.Ad (Ad (..)) + +leboncoin :: [Ad] +leboncoin = + [ Ad {name = "Chambre style hotel", location = "Dinan", price = Just "265 €", url = "https://www.leboncoin.fr/locations/1450271298.htm?ca=6_s"} + , Ad {name = "Centre ville", location = "Brest", price = Just "420 €", url = "https://www.leboncoin.fr/locations/1450271056.htm?ca=6_s"} + , Ad {name = "T3, Morlaix proche centre ville et port plaisance", location = "Morlaix", price = Just "395 €", url = "https://www.leboncoin.fr/locations/1407544500.htm?ca=6_s"} + , Ad {name = "Appartement meuble", location = "Rennes", price = Just "565 €", url = "https://www.leboncoin.fr/locations/1377975959.htm?ca=6_s"} + , Ad {name = "A louer T3", location = "La Chapelle-Janson", price = Just "420 €", url = "https://www.leboncoin.fr/locations/1450270098.htm?ca=6_s"} + , Ad {name = "locations garage / garde meuble", location = "Landerneau", price = Just "50 €", url = "https://www.leboncoin.fr/locations/1450174538.htm?ca=6_s"} + , Ad {name = "Studio meublé", location = "Lanester", price = Just "290 €", url = "https://www.leboncoin.fr/locations/1450268170.htm?ca=6_s"} + , Ad {name = "Ergué-Gabéric - Maison - 3 chambres -", location = "Ergué-Gabéric", price = Just "655 €", url = "https://www.leboncoin.fr/locations/1450267530.htm?ca=6_s"} + , Ad {name = "Studio meublé à 2 mn de la gare", location = "Rennes", price = Just "475 €", url = "https://www.leboncoin.fr/locations/1450267333.htm?ca=6_s"} + , Ad {name = "Chambre à louer chez l,habitant", location = "Rennes", price = Just "450 €", url = "https://www.leboncoin.fr/locations/1450266273.htm?ca=6_s"} + , Ad {name = "Appartement 3 pièces 63 m\178", location = "Brest", price = Just "600 €", url = "https://www.leboncoin.fr/locations/1426927284.htm?ca=6_s"} + , Ad {name = "recherche location", location = "Plaintel", price = Just "550 €", url = "https://www.leboncoin.fr/locations/1450265866.htm?ca=6_s"} + , Ad {name = "Studio rénové - Centre Ville Lorient", location = "Lorient", price = Just "360 €", url = "https://www.leboncoin.fr/locations/1450265154.htm?ca=6_s"} + , Ad {name = "Centre historique, t2 meuble", location = "Vannes", price = Just "440 €", url = "https://www.leboncoin.fr/locations/1450263864.htm?ca=6_s"} + , Ad {name = "appartement t3 parking privé", location = "Pontrieux", price = Just "390 €", url = "https://www.leboncoin.fr/locations/1450251207.htm?ca=6_s"} + , Ad {name = "T1 Bis espace couchage indépendant", location = "Brest", price = Just "360 €", url = "https://www.leboncoin.fr/locations/1450263863.htm?ca=6_s"} + , Ad {name = "Grand T2 boulevard Gambetta", location = "Brest", price = Just "400 €", url = "https://www.leboncoin.fr/locations/1450263306.htm?ca=6_s"} + , Ad {name = "chambre meublée", location = "Bourg-des-Comptes", price = Just "300 €", url = "https://www.leboncoin.fr/locations/1450246530.htm?ca=6_s"} + , Ad {name = "Un studio de charme au centre de Landerneau", location = "Landerneau", price = Just "310 €", url = "https://www.leboncoin.fr/locations/1433376198.htm?ca=6_s"} + , Ad {name = "Appartement T2 plein centre.", location = "Pluméliau", price = Just "385 €", url = "https://www.leboncoin.fr/locations/1450262744.htm?ca=6_s"} + , Ad {name = "Recherche l'appartement parfait", location = "Rennes", price = Just "360 €", url = "https://www.leboncoin.fr/locations/1450260894.htm?ca=6_s"} + , Ad {name = "Location appartement meuble", location = "Saint-Malo", price = Just "510 €", url = "https://www.leboncoin.fr/locations/1421964890.htm?ca=6_s"} + , Ad {name = "Maison neuve Lambezelec", location = "Plabennec", price = Just "900 €", url = "https://www.leboncoin.fr/locations/1450259912.htm?ca=6_s"} + , Ad {name = "Chambre kitchenette chez l'habitant", location = "Brest", price = Just "320 €", url = "https://www.leboncoin.fr/locations/1450258838.htm?ca=6_s"} + , Ad {name = "Studio 20m2 Rennes Ouest", location = "Rennes", price = Just "395 €", url = "https://www.leboncoin.fr/locations/1450258223.htm?ca=6_s"} + , Ad {name = "Maison Ploufragan", location = "Ploufragan", price = Just "680 €", url = "https://www.leboncoin.fr/locations/1450257866.htm?ca=6_s"} + , Ad {name = "Studio centre villes Rennes", location = "Rennes", price = Just "425 €", url = "https://www.leboncoin.fr/locations/1450257277.htm?ca=6_s"} + , Ad {name = "Grand T2 Saint-Brieuc proche Renan", location = "Saint-Brieuc", price = Just "375 €", url = "https://www.leboncoin.fr/locations/1450257042.htm?ca=6_s"} + , Ad {name = "Location maison T3", location = "Langueux", price = Just "569 €", url = "https://www.leboncoin.fr/locations/1439054110.htm?ca=6_s"} + , Ad {name = "Chambre", location = "Vannes", price = Just "150 €", url = "https://www.leboncoin.fr/locations/1450255229.htm?ca=6_s"} + , Ad {name = "Appartement meublé 2 ch centre Brest", location = "Brest", price = Just "750 €", url = "https://www.leboncoin.fr/locations/1450255216.htm?ca=6_s"} + , Ad {name = "Brest Base Navale/porte Caffa - possible meublé", location = "Brest", price = Just "390 €", url = "https://www.leboncoin.fr/locations/1433862279.htm?ca=6_s"} + , Ad {name = "Appartement Rennes", location = "Rennes", price = Just "390 €", url = "https://www.leboncoin.fr/locations/1450254830.htm?ca=6_s"} + , Ad {name = "Location Appartement Villejean", location = "Rennes", price = Just "400 €", url = "https://www.leboncoin.fr/locations/1450253542.htm?ca=6_s"} + , Ad {name = "Maison de bourg 3 chambres Bédée", location = "Bédée", price = Just "570 €", url = "https://www.leboncoin.fr/locations/1450252631.htm?ca=6_s"} + ] + +ouestFrance :: [Ad] +ouestFrance = + [ Ad {name = "Maison 3 chambres", location = "Rennes", price = Just "210 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13108048.htm"} + , Ad {name = "Maison 4 chambres", location = "Rennes Sacré Coeurs", price = Just "1 094 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13114249.htm"} + , Ad {name = "Maison 1 pièce", location = "Rennes Bourg L'evêque", price = Just "370 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13113524.htm"} + , Ad {name = "Maison 2 chambres", location = "Rennes Sainte-Thérèse", price = Just "869 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13111302.htm"} + , Ad {name = "Maison 4 chambres", location = "Rennes Nord Saint-Martin", price = Just "1 650 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13109610.htm"} + , Ad {name = "Maison 2 chambres", location = "Rennes Sainte-Thérèse", price = Just "915 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13020322.htm"} + , Ad {name = "Maison 4 chambres", location = "Rennes Brequigny", price = Just "1 011 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13107308.htm"} + , Ad {name = "Maison 6 chambres", location = "Rennes Jeanne D'arc", price = Just "2 715 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13104531.htm"} + , Ad {name = "Maison 3 chambres", location = "Rennes", price = Just "1 025 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13104556.htm"} + , Ad {name = "Maison 4 chambres", location = "Rennes Patton", price = Just "1 270 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12742357.htm"} + , Ad {name = "Maison 4 chambres", location = "Rennes Jeanne D'arc", price = Just "1 616 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12938970.htm"} + , Ad {name = "Maison 4 chambres", location = "Rennes Sainte-Thérèse", price = Just "1 910 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13050770.htm"} + , Ad {name = "Maison 7 chambres", location = "Rennes Centre Ville", price = Just "3 000 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12888774.htm"} + , Ad {name = "Maison 5 chambres", location = "Rennes Patton", price = Just "1 500 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12317304.htm"} + , Ad {name = "Maison 6 chambres", location = "Rennes Sainte-Thérèse", price = Just "1 850 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12874271.htm"} + , Ad {name = "Maison 4 chambres", location = "Rennes Sud", price = Just "895 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11139843.htm"} + , Ad {name = "Maison 3 chambres", location = "Rennes Cleunay", price = Just "760 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11538097.htm"} + , Ad {name = "Maison 5 chambres", location = "Rennes Sud", price = Just "725 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11538087.htm"} + , Ad {name = "Maison 2 pièces", location = "Rennes Sacré Coeurs", price = Just "420 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12892421.htm"} + , Ad {name = "Maison 2 chambres", location = "Rennes Centre Ville", price = Just "700 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11918621.htm"} + , Ad {name = "Maison 5 pièces", location = "Rennes Arsenal - Redon", price = Just "760 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11139832.htm"} + , Ad {name = "Maison 2 chambres", location = "Rennes Centre Ville", price = Just "900 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12247109.htm"} + , Ad {name = "Maison 4 pièces", location = "Rennes Nord Saint-Martin", price = Just "680 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12973592.htm"} + ] + +seLoger :: [Ad] +seLoger = + [ Ad {name = "Appartement", location = "Paris 20ème", price = Just "1 219 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-20eme-75/plaine/134448375.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 16ème", price = Just "1 500 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-16eme-75/muette-sud/134049627.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 3ème", price = Just "1 595 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-3eme-75/arts-et-metiers/76038037.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 15ème", price = Just "1 612 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-15eme-75/citroen-boucicaut/135038561.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 16ème", price = Just "1 650 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-16eme-75/auteuil-sud/133027227.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 16ème", price = Just "1 875 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-16eme-75/chaillot/127190787.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 8ème", price = Just "1 985 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-8eme-75/saint-philippe-du-roule/127109353.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 1er", price = Just "2 090 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-1er-75/ile-de-la-cite/133982099.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 16ème", price = Just "2 240 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-16eme-75/porte-dauphine/118523093.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 17ème", price = Just "2 400 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-17eme-75/champerret-berthier/134621503.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 6ème", price = Just "2 055 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-6eme-75/saint-germain-des-pres/130192167.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 14ème", price = Just "1 300 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-14eme-75/jean-moulin-porte-d-orleans/133269543.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 16ème", price = Just "3 500 €", url = "https://www.bellesdemeures.com/annonces/locations/appartement/paris-16eme-75/133260359.htm?bd=CartoToList_SL&furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&cmp=INTSL_ListToDetail"} + , Ad {name = "Appartement", location = "Paris 17ème", price = Just "1 890 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-17eme-75/134972009.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 18ème", price = Just "1 390 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-18eme-75/clignancourt-jules-joffrin/134735501.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 6ème", price = Just "1 600 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-6eme-75/saint-germain-des-pres/134437989.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 11ème", price = Just "1 980 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-11eme-75/bastille-popincourt/134314607.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 13ème", price = Just "901 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-13eme-75/olympiades-choisy/134428153.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 12ème", price = Just "1 250 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-12eme-75/aligre-gare-de-lyon/133908165.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + , Ad {name = "Appartement", location = "Paris 17ème", price = Just "1 100 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-17eme-75/legendre-levis/134131479.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} + ] diff --git a/src/test/haskell/Main.hs b/src/test/haskell/Main.hs index fa86d27..a8cfae9 100644 --- a/src/test/haskell/Main.hs +++ b/src/test/haskell/Main.hs @@ -1,6 +1,9 @@ +import Data.Maybe (catMaybes) import qualified Data.Text.IO as T import Test.Hspec +import qualified Ads +import qualified FetchAd import Model.Ad (Ad (..)) import qualified Parser.LeboncoinParser as LeboncoinParser import qualified Parser.OuestFranceParser as OuestFranceParser @@ -8,56 +11,19 @@ import qualified Parser.SeLogerParser as SeLogerParser main :: IO () main = do - rawLeboncoinAds <- T.readFile "src/test/resources/leboncoin.html" - rawOuestFranceAds <- T.readFile "src/test/resources/ouestFrance.html" - rawSeLogerParser <- T.readFile "src/test/resources/seLoger.html" - hspec $ do describe "LeboncoinParser" $ do it "should parse no results from empty string" $ do LeboncoinParser.parse "" `shouldBe` [] - it "should parse ads from page" $ do - let ads = - [ Ad {name = "Chambre style hotel", location = "Dinan", price = Just "265 €", url = "https://www.leboncoin.fr/locations/1450271298.htm?ca=6_s"} - , Ad {name = "Centre ville", location = "Brest", price = Just "420 €", url = "https://www.leboncoin.fr/locations/1450271056.htm?ca=6_s"} - , Ad {name = "T3, Morlaix proche centre ville et port plaisance", location = "Morlaix", price = Just "395 €", url = "https://www.leboncoin.fr/locations/1407544500.htm?ca=6_s"} - , Ad {name = "Appartement meuble", location = "Rennes", price = Just "565 €", url = "https://www.leboncoin.fr/locations/1377975959.htm?ca=6_s"} - , Ad {name = "A louer T3", location = "La Chapelle-Janson", price = Just "420 €", url = "https://www.leboncoin.fr/locations/1450270098.htm?ca=6_s"} - , Ad {name = "locations garage / garde meuble", location = "Landerneau", price = Just "50 €", url = "https://www.leboncoin.fr/locations/1450174538.htm?ca=6_s"} - , Ad {name = "Studio meublé", location = "Lanester", price = Just "290 €", url = "https://www.leboncoin.fr/locations/1450268170.htm?ca=6_s"} - , Ad {name = "Ergué-Gabéric - Maison - 3 chambres -", location = "Ergué-Gabéric", price = Just "655 €", url = "https://www.leboncoin.fr/locations/1450267530.htm?ca=6_s"} - , Ad {name = "Studio meublé à 2 mn de la gare", location = "Rennes", price = Just "475 €", url = "https://www.leboncoin.fr/locations/1450267333.htm?ca=6_s"} - , Ad {name = "Chambre à louer chez l,habitant", location = "Rennes", price = Just "450 €", url = "https://www.leboncoin.fr/locations/1450266273.htm?ca=6_s"} - , Ad {name = "Appartement 3 pièces 63 m\178", location = "Brest", price = Just "600 €", url = "https://www.leboncoin.fr/locations/1426927284.htm?ca=6_s"} - , Ad {name = "recherche location", location = "Plaintel", price = Just "550 €", url = "https://www.leboncoin.fr/locations/1450265866.htm?ca=6_s"} - , Ad {name = "Studio rénové - Centre Ville Lorient", location = "Lorient", price = Just "360 €", url = "https://www.leboncoin.fr/locations/1450265154.htm?ca=6_s"} - , Ad {name = "Centre historique, t2 meuble", location = "Vannes", price = Just "440 €", url = "https://www.leboncoin.fr/locations/1450263864.htm?ca=6_s"} - , Ad {name = "appartement t3 parking privé", location = "Pontrieux", price = Just "390 €", url = "https://www.leboncoin.fr/locations/1450251207.htm?ca=6_s"} - , Ad {name = "T1 Bis espace couchage indépendant", location = "Brest", price = Just "360 €", url = "https://www.leboncoin.fr/locations/1450263863.htm?ca=6_s"} - , Ad {name = "Grand T2 boulevard Gambetta", location = "Brest", price = Just "400 €", url = "https://www.leboncoin.fr/locations/1450263306.htm?ca=6_s"} - , Ad {name = "chambre meublée", location = "Bourg-des-Comptes", price = Just "300 €", url = "https://www.leboncoin.fr/locations/1450246530.htm?ca=6_s"} - , Ad {name = "Un studio de charme au centre de Landerneau", location = "Landerneau", price = Just "310 €", url = "https://www.leboncoin.fr/locations/1433376198.htm?ca=6_s"} - , Ad {name = "Appartement T2 plein centre.", location = "Pluméliau", price = Just "385 €", url = "https://www.leboncoin.fr/locations/1450262744.htm?ca=6_s"} - , Ad {name = "Recherche l'appartement parfait", location = "Rennes", price = Just "360 €", url = "https://www.leboncoin.fr/locations/1450260894.htm?ca=6_s"} - , Ad {name = "Location appartement meuble", location = "Saint-Malo", price = Just "510 €", url = "https://www.leboncoin.fr/locations/1421964890.htm?ca=6_s"} - , Ad {name = "Maison neuve Lambezelec", location = "Plabennec", price = Just "900 €", url = "https://www.leboncoin.fr/locations/1450259912.htm?ca=6_s"} - , Ad {name = "Chambre kitchenette chez l'habitant", location = "Brest", price = Just "320 €", url = "https://www.leboncoin.fr/locations/1450258838.htm?ca=6_s"} - , Ad {name = "Studio 20m2 Rennes Ouest", location = "Rennes", price = Just "395 €", url = "https://www.leboncoin.fr/locations/1450258223.htm?ca=6_s"} - , Ad {name = "Maison Ploufragan", location = "Ploufragan", price = Just "680 €", url = "https://www.leboncoin.fr/locations/1450257866.htm?ca=6_s"} - , Ad {name = "Studio centre villes Rennes", location = "Rennes", price = Just "425 €", url = "https://www.leboncoin.fr/locations/1450257277.htm?ca=6_s"} - , Ad {name = "Grand T2 Saint-Brieuc proche Renan", location = "Saint-Brieuc", price = Just "375 €", url = "https://www.leboncoin.fr/locations/1450257042.htm?ca=6_s"} - , Ad {name = "Location maison T3", location = "Langueux", price = Just "569 €", url = "https://www.leboncoin.fr/locations/1439054110.htm?ca=6_s"} - , Ad {name = "Chambre", location = "Vannes", price = Just "150 €", url = "https://www.leboncoin.fr/locations/1450255229.htm?ca=6_s"} - , Ad {name = "Appartement meublé 2 ch centre Brest", location = "Brest", price = Just "750 €", url = "https://www.leboncoin.fr/locations/1450255216.htm?ca=6_s"} - , Ad {name = "Brest Base Navale/porte Caffa - possible meublé", location = "Brest", price = Just "390 €", url = "https://www.leboncoin.fr/locations/1433862279.htm?ca=6_s"} - , Ad {name = "Appartement Rennes", location = "Rennes", price = Just "390 €", url = "https://www.leboncoin.fr/locations/1450254830.htm?ca=6_s"} - , Ad {name = "Location Appartement Villejean", location = "Rennes", price = Just "400 €", url = "https://www.leboncoin.fr/locations/1450253542.htm?ca=6_s"} - , Ad {name = "Maison de bourg 3 chambres Bédée", location = "Bédée", price = Just "570 €", url = "https://www.leboncoin.fr/locations/1450252631.htm?ca=6_s"} - ] + it "should parse ads from local page" $ do + ads <- T.readFile "src/test/resources/leboncoin.html" + LeboncoinParser.parse ads `shouldBe` Ads.leboncoin - LeboncoinParser.parse rawLeboncoinAds `shouldBe` ads + it "should parse ads from remote page" $ do + ads <- FetchAd.leboncoin ["https://www.leboncoin.fr/locations/offres/ile_de_france/?th=1"] + checkAds ads describe "OuestFranceParser" $ do @@ -65,33 +31,12 @@ main = do OuestFranceParser.parse "" `shouldBe` [] it "should parse ads from page" $ do - let ads = - [ Ad {name = "Maison 3 chambres", location = "Rennes", price = Just "210 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13108048.htm"} - , Ad {name = "Maison 4 chambres", location = "Rennes Sacré Coeurs", price = Just "1 094 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13114249.htm"} - , Ad {name = "Maison 1 pièce", location = "Rennes Bourg L'evêque", price = Just "370 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13113524.htm"} - , Ad {name = "Maison 2 chambres", location = "Rennes Sainte-Thérèse", price = Just "869 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13111302.htm"} - , Ad {name = "Maison 4 chambres", location = "Rennes Nord Saint-Martin", price = Just "1 650 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13109610.htm"} - , Ad {name = "Maison 2 chambres", location = "Rennes Sainte-Thérèse", price = Just "915 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13020322.htm"} - , Ad {name = "Maison 4 chambres", location = "Rennes Brequigny", price = Just "1 011 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13107308.htm"} - , Ad {name = "Maison 6 chambres", location = "Rennes Jeanne D'arc", price = Just "2 715 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13104531.htm"} - , Ad {name = "Maison 3 chambres", location = "Rennes", price = Just "1 025 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13104556.htm"} - , Ad {name = "Maison 4 chambres", location = "Rennes Patton", price = Just "1 270 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12742357.htm"} - , Ad {name = "Maison 4 chambres", location = "Rennes Jeanne D'arc", price = Just "1 616 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12938970.htm"} - , Ad {name = "Maison 4 chambres", location = "Rennes Sainte-Thérèse", price = Just "1 910 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/13050770.htm"} - , Ad {name = "Maison 7 chambres", location = "Rennes Centre Ville", price = Just "3 000 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12888774.htm"} - , Ad {name = "Maison 5 chambres", location = "Rennes Patton", price = Just "1 500 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12317304.htm"} - , Ad {name = "Maison 6 chambres", location = "Rennes Sainte-Thérèse", price = Just "1 850 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12874271.htm"} - , Ad {name = "Maison 4 chambres", location = "Rennes Sud", price = Just "895 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11139843.htm"} - , Ad {name = "Maison 3 chambres", location = "Rennes Cleunay", price = Just "760 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11538097.htm"} - , Ad {name = "Maison 5 chambres", location = "Rennes Sud", price = Just "725 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11538087.htm"} - , Ad {name = "Maison 2 pièces", location = "Rennes Sacré Coeurs", price = Just "420 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12892421.htm"} - , Ad {name = "Maison 2 chambres", location = "Rennes Centre Ville", price = Just "700 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11918621.htm"} - , Ad {name = "Maison 5 pièces", location = "Rennes Arsenal - Redon", price = Just "760 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/11139832.htm"} - , Ad {name = "Maison 2 chambres", location = "Rennes Centre Ville", price = Just "900 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12247109.htm"} - , Ad {name = "Maison 4 pièces", location = "Rennes Nord Saint-Martin", price = Just "680 €", url = "https://www.ouestfrance-immo.com//immobilier/location/maison/rennes-35-35238/12973592.htm"} - ] + rawOuestFranceAds <- T.readFile "src/test/resources/ouestFrance.html" + OuestFranceParser.parse rawOuestFranceAds `shouldBe` Ads.ouestFrance - OuestFranceParser.parse rawOuestFranceAds `shouldBe` ads + it "should parse ads from remote page" $ do + ads <- FetchAd.ouestFrance ["https://www.ouestfrance-immo.com/louer/appartement/rennes-35-35000/"] + checkAds ads describe "SeLogerParser" $ do @@ -99,27 +44,14 @@ main = do SeLogerParser.parse "" `shouldBe` [] it "should parse ads from page" $ do - let ads = - [ Ad {name = "Appartement", location = "Paris 20ème", price = Just "1 219 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-20eme-75/plaine/134448375.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 16ème", price = Just "1 500 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-16eme-75/muette-sud/134049627.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 3ème", price = Just "1 595 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-3eme-75/arts-et-metiers/76038037.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 15ème", price = Just "1 612 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-15eme-75/citroen-boucicaut/135038561.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 16ème", price = Just "1 650 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-16eme-75/auteuil-sud/133027227.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 16ème", price = Just "1 875 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-16eme-75/chaillot/127190787.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 8ème", price = Just "1 985 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-8eme-75/saint-philippe-du-roule/127109353.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 1er", price = Just "2 090 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-1er-75/ile-de-la-cite/133982099.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 16ème", price = Just "2 240 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-16eme-75/porte-dauphine/118523093.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 17ème", price = Just "2 400 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-17eme-75/champerret-berthier/134621503.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 6ème", price = Just "2 055 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-6eme-75/saint-germain-des-pres/130192167.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 14ème", price = Just "1 300 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-14eme-75/jean-moulin-porte-d-orleans/133269543.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 16ème", price = Just "3 500 €", url = "https://www.bellesdemeures.com/annonces/locations/appartement/paris-16eme-75/133260359.htm?bd=CartoToList_SL&furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&cmp=INTSL_ListToDetail"} - , Ad {name = "Appartement", location = "Paris 17ème", price = Just "1 890 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-17eme-75/134972009.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 18ème", price = Just "1 390 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-18eme-75/clignancourt-jules-joffrin/134735501.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 6ème", price = Just "1 600 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-6eme-75/saint-germain-des-pres/134437989.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 11ème", price = Just "1 980 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-11eme-75/bastille-popincourt/134314607.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 13ème", price = Just "901 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-13eme-75/olympiades-choisy/134428153.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 12ème", price = Just "1 250 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-12eme-75/aligre-gare-de-lyon/133908165.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - , Ad {name = "Appartement", location = "Paris 17ème", price = Just "1 100 €", url = "https://www.seloger.com/annonces/locations/appartement/paris-17eme-75/legendre-levis/134131479.htm?furnished=0&places=%5b%7bcp%3a75%7d%5d&projects=1&qsversion=1.0&rooms=2&types=1&bd=ListToDetail"} - ] + ads <- T.readFile "src/test/resources/seLoger.html" + SeLogerParser.parse ads `shouldBe` Ads.seLoger + + it "should parse ads from remote page" $ do + ads <- FetchAd.seLoger ["https://www.seloger.com/list.htm?tri=initial&idtypebien=2,1&idtt=2,5&naturebien=1,2,4&ci=690123"] + checkAds ads - SeLogerParser.parse rawSeLogerParser `shouldBe` ads +checkAds :: [Ad] -> IO () +checkAds ads = do + length ads `shouldSatisfy` (\n -> n > 10) + (length . catMaybes . map price $ ads) `shouldSatisfy` (\n -> n > 10) -- cgit v1.2.3