From cf69372204ac3b02c6762af295e20242d5dd5c47 Mon Sep 17 00:00:00 2001
From: Guntis Smaukstelis <gsmaukstelis@evolution.com>
Date: Mon, 10 Feb 2025 23:43:58 +0200
Subject: [PATCH] Download recent forecasts with checking local files

---
 src/main/scala/data/DataService.scala         |  6 ++--
 src/main/scala/fetchDMI/FetchService.scala    | 28 +++++++++++++++++--
 .../scala/fetchDMI/FetchServiceTest.scala     | 22 ++++++++++++++-
 3 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/src/main/scala/data/DataService.scala b/src/main/scala/data/DataService.scala
index 1578872..bbad0fb 100644
--- a/src/main/scala/data/DataService.scala
+++ b/src/main/scala/data/DataService.scala
@@ -4,7 +4,7 @@ import cats.effect.IO
 import fs2.io.file.{Files, Path}
 import grib.{Grib, GribParser}
 
-import java.time.{ZoneId, ZonedDateTime}
+import java.time.{ZoneId, ZoneOffset, ZonedDateTime}
 import java.time.format.DateTimeFormatter
 import scala.io.Source
 import scala.util.Try
@@ -54,8 +54,8 @@ object DataService {
       val start = filename.slice(9, 25) // gets "2025-02-05T1500Z"
       val end = filename.slice(26, 42) // gets "2025-02-05T1800Z"
       (
-        ZonedDateTime.parse(start, formatter),
-        ZonedDateTime.parse(end, formatter)
+        ZonedDateTime.parse(start, formatter).toInstant.atZone(ZoneOffset.UTC),
+        ZonedDateTime.parse(end, formatter).toInstant.atZone(ZoneOffset.UTC)
       )
     }.toOption
   }
diff --git a/src/main/scala/fetchDMI/FetchService.scala b/src/main/scala/fetchDMI/FetchService.scala
index 466811f..c462f41 100644
--- a/src/main/scala/fetchDMI/FetchService.scala
+++ b/src/main/scala/fetchDMI/FetchService.scala
@@ -87,12 +87,34 @@ class FetchService(log: Logger[IO]) {
         gribTime = gribList.head.time
         fileName = Path(s"${DataService.FOLDER}/harmonie_${gribTime.referenceTime}_${gribTime.forecastTime}.grib".replace(":", ""))
         _ <- Files[IO].move(tmpPath, fileName, CopyFlags.apply(CopyFlag.ReplaceExisting))
-        _ <- log.info(fileName.toString)
-        fileNameStr = fileName.toString
-      } yield fileNameStr
+        fileSizeBytes <- Files[IO].size(fileName)
+        fileSizeMB = fileSizeBytes.toDouble / (1024 * 1024)
+        _ <- log.info(s" ${"%.1f".format(fileSizeMB)} MB - ${fileName.fileName}")
+      } yield fileName.toString
     }
   }
 
+  /**
+   * get latest model run from STAC API
+   * check local forecast grib files not to download them again
+   * fetch those forecasts
+   */
+  def fetchRecentForecasts(): IO[Unit] = {
+    for {
+      dateTimeList <- generateFetchList()
+      resultList <- fetchFromList(dateTimeList)
+    } yield ()
+  }
+
+  def generateFetchList(): IO[List[ZonedDateTime]] = {
+    for {
+      availableResult <- fetchAvailableForecasts()
+      (modelRun, forecastDateList) = availableResult
+      localForecasts <- DataService.getForecasts()
+      toFetchList = forecastDateList.filter(dateTime => !localForecasts.contains((modelRun, dateTime)))
+    } yield toFetchList
+  }
+
   def fetchAvailableForecasts(): IO[(ZonedDateTime, List[ZonedDateTime])] = {
     EmberClientBuilder.default[IO].build.use { client =>
       for {
diff --git a/src/main/scala/fetchDMI/FetchServiceTest.scala b/src/main/scala/fetchDMI/FetchServiceTest.scala
index 6b2aace..b3dde79 100644
--- a/src/main/scala/fetchDMI/FetchServiceTest.scala
+++ b/src/main/scala/fetchDMI/FetchServiceTest.scala
@@ -8,7 +8,27 @@ import java.time.{ZoneOffset, ZonedDateTime}
 object FetchServiceTest {
   def main(args: Array[String]): Unit = {
 //    fetchFromTimeList().unsafeRunSync()
-    fetchAvailableForecasts.unsafeRunSync()
+//    fetchAvailableForecasts.unsafeRunSync()
+//    generateFetchList().unsafeRunSync()
+    fetchRecentForecasts().unsafeRunSync()
+  }
+
+  private def fetchRecentForecasts(): IO[Unit] = {
+    val program = for {
+      fetch <- FetchService.of
+      result <- fetch.fetchRecentForecasts()
+      _ <- IO.println("-=fetch finished=-")
+    } yield ()
+    program
+  }
+
+  private def generateFetchList(): IO[Unit] = {
+    val program = for {
+      fetch <- FetchService.of
+      list <- fetch.generateFetchList()
+      _ <- IO.println(list)
+    } yield ()
+    program
   }
 
   private def fetchAvailableForecasts(): IO[Unit] = {