diff options
Diffstat (limited to 'src/lib/haskell/Parser')
-rw-r--r-- | src/lib/haskell/Parser/LeboncoinParser.hs | 24 | ||||
-rw-r--r-- | src/lib/haskell/Parser/OuestFranceParser.hs | 25 | ||||
-rw-r--r-- | src/lib/haskell/Parser/SeLogerParser.hs | 24 | ||||
-rw-r--r-- | src/lib/haskell/Parser/Utils.hs | 48 |
4 files changed, 121 insertions, 0 deletions
diff --git a/src/lib/haskell/Parser/LeboncoinParser.hs b/src/lib/haskell/Parser/LeboncoinParser.hs new file mode 100644 index 0000000..77213cb --- /dev/null +++ b/src/lib/haskell/Parser/LeboncoinParser.hs @@ -0,0 +1,24 @@ +module Parser.LeboncoinParser + ( parse + ) where + +import Data.Maybe (catMaybes) +import Data.Text (Text) +import qualified Data.Text as T +import Text.HTML.TagSoup + +import Model.Ad (Ad (Ad)) +import Parser.Utils + +parse :: Text -> [Ad] +parse page = + catMaybes . fmap parseAd $ partitions (~== (T.unpack "<a>")) tags + where tags = getTagsBetween "<li itemtype=http://schema.org/Offer>" "<div class=information-immo_content>" (parseTags page) + +parseAd :: [Tag Text] -> Maybe Ad +parseAd tags = do + name <- getTagTextAfter "<h2 class=item_title>" tags + location <- getTagAttribute "<meta itemprop=address>" "content" tags + let price = getTagTextAfter "<h3 class=item_price>" tags + url <- getTagAttribute "<a>" "href" tags + return (Ad name location price (T.concat ["https:", url])) diff --git a/src/lib/haskell/Parser/OuestFranceParser.hs b/src/lib/haskell/Parser/OuestFranceParser.hs new file mode 100644 index 0000000..f46ed03 --- /dev/null +++ b/src/lib/haskell/Parser/OuestFranceParser.hs @@ -0,0 +1,25 @@ +module Parser.OuestFranceParser + ( parse + ) where + +import Data.Maybe (catMaybes) +import Data.Text (Text) +import qualified Data.Text as T +import Text.HTML.TagSoup + +import Model.Ad (Ad (Ad)) +import Parser.Utils + +parse :: Text -> [Ad] +parse page = + catMaybes . fmap parseAd $ partitions (~== (T.unpack "<a>")) tags + where tags = getTagsBetween "<div id=listAnnonces>" "<div id=interactions>" (parseTags page) + +parseAd :: [Tag Text] -> Maybe Ad +parseAd tags = do + name <- getTagTextAfter "<span class=annTitre>" tags + location <- getTagTextAfter "<span class=annVille>" tags + let price = getTagTextAfter "<span class=annPrix>" tags + let startUrl = "https://www.ouestfrance-immo.com/" + url <- getTagAttribute "<a>" "href" tags + return (Ad name location price (T.concat [startUrl, url])) diff --git a/src/lib/haskell/Parser/SeLogerParser.hs b/src/lib/haskell/Parser/SeLogerParser.hs new file mode 100644 index 0000000..b073862 --- /dev/null +++ b/src/lib/haskell/Parser/SeLogerParser.hs @@ -0,0 +1,24 @@ +module Parser.SeLogerParser + ( parse + ) where + +import Data.Maybe (catMaybes) +import Data.Text (Text) +import qualified Data.Text as T +import Text.HTML.TagSoup + +import Model.Ad (Ad (Ad)) +import Parser.Utils + +parse :: Text -> [Ad] +parse page = + catMaybes . fmap parseAd $ partitions (~== (T.unpack "<div class=c-pa-info>")) tags + where tags = getTagsBetween "<section class=liste_resultat>" "<div class=bottomAnchorWrapper>" (parseTags page) + +parseAd :: [Tag Text] -> Maybe Ad +parseAd tags = do + name <- getTagTextAfter "<a>" tags + location <- getTagTextAfter "<div class=c-pa-city>" tags + let price = getTagTextAfter "<span class=c-pa-cprice>" tags + url <- getTagAttribute "<a>" "href" tags + return (Ad name location price url) diff --git a/src/lib/haskell/Parser/Utils.hs b/src/lib/haskell/Parser/Utils.hs new file mode 100644 index 0000000..4768327 --- /dev/null +++ b/src/lib/haskell/Parser/Utils.hs @@ -0,0 +1,48 @@ +module Parser.Utils + ( getTagsBefore + , getTagsAfter + , getTagsBetween + , getTagAttributes + , getTagAttribute + , getTagTextAfter + ) where + +import Data.List (find, findIndex) +import Data.Maybe (catMaybes, listToMaybe) +import Data.Text (Text) +import qualified Data.Text as T +import Text.HTML.TagSoup + +getTagsBefore :: Text -> [Tag Text] -> [Tag Text] +getTagsBefore selector = takeWhile (~/= (T.unpack selector)) + +getTagsAfter :: Text -> [Tag Text] -> [Tag Text] +getTagsAfter selector = drop 1 . dropWhile (~/= (T.unpack selector)) + +getTagsBetween :: Text -> Text -> [Tag Text] -> [Tag Text] +getTagsBetween begin end = getTagsBefore end . getTagsAfter begin + +getTagAttributes :: Text -> Text -> [Tag Text] -> [Text] +getTagAttributes selector attribute = + catMaybes + . fmap (maybeTagAttribute attribute) + . filter (~== (T.unpack selector)) + +getTagAttribute :: Text -> Text -> [Tag Text] -> Maybe Text +getTagAttribute selector attribute = + listToMaybe + . getTagAttributes selector attribute + +getTagTextAfter :: Text -> [Tag Text] -> Maybe Text +getTagTextAfter selector tags = + case findIndex (~== (T.unpack selector)) tags of + Just index -> fmap T.strip $ safeGetAt (index + 1) tags >>= maybeTagText + Nothing -> Nothing + +maybeTagAttribute :: Text -> Tag Text -> Maybe Text +maybeTagAttribute name (TagOpen _ xs) = + fmap snd . find (\(x, _) -> x == name) $ xs +maybeTagAttribute _ _ = Nothing + +safeGetAt :: Int -> [a] -> Maybe a +safeGetAt index = listToMaybe . drop index |