Server fetches and saves files. Also proper error handling

src/main/scala/fetch/FetchData.scala

@@ -1,42 +0,0 @@
-package fetch
-
-import cats.effect._
-import cats.implicits._
-import org.http4s._
-import org.http4s.blaze.client.BlazeClientBuilder
-import org.http4s.client.Client
-import org.http4s.headers.Authorization
-
-import com.typesafe.config.ConfigFactory
-
-import java.nio.file.{Files, Paths}
-import scala.concurrent.ExecutionContext.global
-
-object FetchData extends IOApp.Simple {
-  // TODO wrap config in IO
-  private val config = ConfigFactory.load()
-  private val basicCredentials = BasicCredentials(config.getString("username"), config.getString("password"))
-  private val baseUrl = Uri.unsafeFromString(config.getString("url")) // 20220831_1330.csv
-
-  def makeRequest(client: Client[IO], url: Uri): IO[Unit] = {
-    val fileName = url.path.toString()
-    for {
-      request <- Request[IO](Method.GET, url)
-        .withHeaders(Authorization(basicCredentials))
-        .pure[IO]
-      responseOrError <- client.expect[String](request).attempt
-      _ <- responseOrError match {
-        case Right(response) => db.DBService.save(fileName, response)
-        case Left(error) => IO(println(s"Request failed to url: $url with error: ${error.getMessage}"))
-      }
-    } yield ()
-  }
-
-  def run: IO[Unit] = {
-    val fileNames = FileName.generateLastNHours(10)
-    val urls = fileNames.map(baseUrl / _)
-    BlazeClientBuilder[IO](global).resource.use { client =>
-      urls.traverse(url => makeRequest(client, url)) // urls.map(...).sequence
-    }
-  }.as(ExitCode.Success)
-}

src/main/scala/fetch/FetchService.scala

@@ -0,0 +1,52 @@
+package fetch
+
+import cats.effect._
+import cats.implicits._
+import org.http4s._
+import org.http4s.blaze.client.BlazeClientBuilder
+import org.http4s.client.Client
+import org.http4s.headers.Authorization
+import com.typesafe.config.ConfigFactory
+
+import java.time.{LocalDate, LocalDateTime}
+import scala.concurrent.ExecutionContext.global
+
+object FetchService {
+  // TODO wrap config in IO
+  private val config = ConfigFactory.load()
+  private val basicCredentials = BasicCredentials(config.getString("username"), config.getString("password"))
+  private val baseUrl = Uri.unsafeFromString(config.getString("url")) // 20220831_1330.csv
+
+  private def makeRequest(client: Client[IO], url: Uri): IO[Either[Throwable, (String, String)]] = {
+    val fileName = url.path.toString()
+    for {
+      request <- Request[IO](Method.GET, url)
+        .withHeaders(Authorization(basicCredentials))
+        .pure[IO]
+      result <- client.expect[String](request).redeemWith(
+        error => IO(Left(error))
+//          .flatTap(_ => IO.println(s"Request failed to url: $url with error: ${error.getMessage}"))
+        ,
+        fileContent => IO(Right((fileName, fileContent)))
+//          .flatTap(_ => IO.println(s"Fetched: $fileName"))
+      )
+    } yield result
+  }
+
+  private def fetchFiles(fileNames: List[String]): IO[List[Either[Throwable, (String, String)]]] = {
+    val urls = fileNames.map(baseUrl / _)
+    BlazeClientBuilder[IO](global).resource.use { client =>
+      urls.traverse(url => makeRequest(client, url))
+    }
+  }
+
+  def fetchInRange(from: LocalDateTime, to: LocalDateTime): IO[List[Either[Throwable, (String, String)]]] = {
+    val fileNames = FileNameService.generate(from, to)
+    fetchFiles(fileNames)
+  }
+
+  def fetchFromDate(date: LocalDate): IO[List[Either[Throwable, (String, String)]]] = {
+    val fileNames = FileNameService.generateFromDate(date)
+    fetchFiles(fileNames)
+  }
+}

src/main/scala/fetch/FileNameService.scala

@@ -1,24 +1,24 @@
 package fetch
 
 import java.time.format.DateTimeFormatter
-import java.time.{LocalDateTime, Duration}
+import java.time.{Duration, LocalDate, LocalDateTime}
 
-object FileName {
+object FileNameService {
   private val interval = Duration.ofMinutes(30)
   private val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
 
-  def generateLastHour(): List[String] = {
-    // TODO IO.timer() IOTimer
-    val now = LocalDateTime.now
-    generate(now.minusHours(1), now)
-  }
-
   // TODO side effect
   def generateLastNHours(hours: Int, now: LocalDateTime = LocalDateTime.now): List[String] = {
     val roundNow = roundToInterval(now, false)
     generate(roundNow.minusHours(hours), roundNow)
   }
 
+  def generateFromDate(date: LocalDate): List[String] = {
+    val from = LocalDateTime.of(date.getYear, date.getMonth, date.getDayOfMonth, 0, 0)
+    val to = LocalDateTime.of(date.getYear, date.getMonth, date.getDayOfMonth, 23, 59)
+    generate(from, to)
+  }
+
   private def roundToInterval(time: LocalDateTime, roundUp: Boolean): LocalDateTime = {
     val minutes = time.getMinute
     val adjustment = if (roundUp) (interval.toMinutes - minutes % interval.toMinutes) % interval.toMinutes

src/main/scala/fetch/Main.scala

@@ -0,0 +1,27 @@
+package fetch
+
+import cats.effect.IO
+import cats.effect.unsafe.implicits.global
+import cats.implicits.toTraverseOps
+import db.DBService
+
+import java.time.{LocalDate, LocalDateTime}
+
+object Main {
+  def run: IO[Unit] = {
+    val from = LocalDateTime.of(2023, 4, 23, 20, 0)
+    val to = LocalDateTime.of(2023, 4, 23, 23, 30)
+    for {
+      fetched <- FetchService.fetchInRange(from, to)
+//      fetched <- FetchService.fetchFromDate(LocalDate.of(2023, 4, 23))
+      (errors, successfulDownloads) = fetched.partitionMap(identity)
+      _ <- IO.println(s"Fetch errors: $errors")
+      saveResults <- successfulDownloads.traverse { case (name, content) => DBService.save(name, content) }
+      _ <- IO.println(s"Save results: $saveResults")
+    } yield ()
+  }
+
+  def main(args: Array[String]): Unit = {
+    run.unsafeRunSync()
+  }
+}