diff --git a/app/Main.hs b/app/Main.hs index 80e71e37673b217be128c3a6a756e481470def50..fd4bc063d2ee6e0d4643c9f096cc2384dd7f7a3e 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -16,16 +16,14 @@ offerFeed = do otodomOffers <- flatScrapOtodomOffers "https://www.otodom.pl/wynajem/mieszkanie/krakow/?search%5Bfilter_float_price%3Ato%5D=3000&search%5Bfilter_enum_rooms_num%5D%5B0%5D=3&search%5Bfilter_enum_rooms_num%5D%5B1%5D=4&search%5Bdist%5D=0&search%5Bsubregion_id%5D=410&search%5Bcity_id%5D=38&search%5Border%5D=created_at_first%3Adesc" gratkaOffers <- scrapGratkaOffers "https://gratka.pl/nieruchomosci/mieszkania/krakow/wynajem?liczba-pokoi:min=3&liczba-pokoi:max=4&cena-calkowita:max=3000&sort=newest" offers <- return . concat $ catMaybes [otodomOffers, gratkaOffers] - timeZone <- getCurrentTimeZone - currentUTCTime <- getCurrentTime - syncedOffers <- syncLastVisitTimestamps (utcToLocalTime timeZone currentUTCTime) offers + syncedOffers <- getCurrentTime >>= \t -> syncLastVisitTimestamps t offers return $ renderOfferFeed syncedOffers saveNewsfeed :: IO () saveNewsfeed = do feed <- offerFeed case feed of - Just x -> T.writeFile "/tmp/newsfeed.xml" x + Just x -> T.writeFile "newsfeed.xml" x Nothing -> Prelude.putStrLn "Scrap failed" main :: IO () diff --git a/src/Newsfeed.hs b/src/Newsfeed.hs index 8fa6f0d41bae249f3386e6d0c206220b79d25dd9..bb7c51320399158d045299659b7ac03ec2e48648 100644 --- a/src/Newsfeed.hs +++ b/src/Newsfeed.hs @@ -15,13 +15,14 @@ import Offer renderOfferFeedEntry :: Offer -> RSSItem renderOfferFeedEntry offer = (nullItem $ offerTitle offer) { rssItemLink = Just $ offerURL offer - , rssItemPubDate = (<> "+0200") <$> (T.pack . formatTime defaultTimeLocale rfc822DateFormat <$> offerVisit offer) + , rssItemPubDate = T.pack . formatTime defaultTimeLocale rfc822DateFormat <$> offerVisit offer } -- FIXME nullRSS empty strings replaced later renderOfferFeed :: [Offer] -> Maybe Text renderOfferFeed offers = - textRSS $ (nullRSS "" "") {rssChannel = (nullChannel "flatscraper" "http://heap.memleak.pl/mmos/newsfeed.xml") + textRSS $ (nullRSS "" "") {rssChannel = (nullChannel "flatscraper" url) {rssItems = renderOfferFeedEntry <$> offers} } + where url = "http://heap.memleak.pl/mmos/newsfeed.xml" diff --git a/src/Offer.hs b/src/Offer.hs index 58ac57963faee38c6d4170c606f4148e07a5823a..e19a6fad51f0369356906e0d2073b74301c42c1e 100644 --- a/src/Offer.hs +++ b/src/Offer.hs @@ -3,11 +3,11 @@ module Offer ) where import Data.Text -import Data.Time.LocalTime +import Data.Time (UTCTime) data Offer = Offer { offerTitle :: Text , offerPriceStr :: Text , offerRentPriceStr :: Maybe Text , offerURL :: Text - , offerVisit :: Maybe LocalTime + , offerVisit :: Maybe UTCTime } deriving (Show, Eq) diff --git a/src/ScrapePersistence.hs b/src/ScrapePersistence.hs index 0e721993f749a59b2d84bc3330885f7e188f2c5b..51022afa65a79c084f6c5bdbbab18b1f80b2c9ea 100644 --- a/src/ScrapePersistence.hs +++ b/src/ScrapePersistence.hs @@ -11,7 +11,6 @@ module ScrapePersistence ( syncLastVisitTimestamps ) where -import Control.Monad.IO.Class (liftIO) import Database.Persist import Database.Persist.Sql import Database.Persist.Sqlite @@ -29,15 +28,13 @@ OfferVisit deriving Show |] -syncLastVisitTimestamps :: LocalTime -> [Offer] -> IO [Offer] +syncLastVisitTimestamps :: UTCTime -> [Offer] -> IO [Offer] syncLastVisitTimestamps timestamp offers = do - -- tz <- getCurrentTimeZone - tz <- return utc - x <- runSqlite "/tmp/szkola_lite.sqlite" $ do + x <- runSqlite "flatscraper.sqlite" $ do runMigration migrateAll - mapM (insertBy . (offerVisit tz)) urls - return $ Prelude.zipWith (zipper tz) x offers + mapM (insertBy . offerVisit) urls + return $ Prelude.zipWith zipper x offers where urls = offerURL <$> offers - offerVisit tz url = OfferVisit url ((localTimeToUTC tz) timestamp) - zipper tz (Left e) r = r {offerVisit = Just $ utcToLocalTime tz $ (offerVisitScrapeTimestamp . entityVal) e} - zipper _ (Right _) r = r {offerVisit = Just $ timestamp} + offerVisit url = OfferVisit url timestamp + zipper (Left e) r = r {offerVisit = Just $ (offerVisitScrapeTimestamp . entityVal) e} + zipper (Right _) r = r {offerVisit = Just $ timestamp}