Add conf into IO and handle IO throwable

src/main/scala/fetch/FetchService.scala

@@ -12,31 +12,39 @@ import java.time.{LocalDate, LocalDateTime}
 import scala.concurrent.ExecutionContext.global
 
 object FetchService {
-  // TODO wrap config in IO
   private val config = ConfigFactory.load()
-  private val basicCredentials = BasicCredentials(config.getString("username"), config.getString("password"))
+  private def getIOString(path: String): IO[String] =
-  private val baseUrl = Uri.unsafeFromString(config.getString("url")) // 20220831_1330.csv
+    if (config.hasPath(path)) IO.pure(config.getString(path))
+    else IO.raiseError(new RuntimeException(s"Missing configuration: $path"))
+
+  private val basicCredentialsIO: IO[BasicCredentials] =
+    for {
+      username <- getIOString("username")
+      password <- getIOString("password")
+    } yield BasicCredentials(username, password)
+
+  private val baseUrlIO: IO[Uri] = getIOString("url").map(Uri.unsafeFromString) // 20220831_1330.csv
 
   private def makeRequest(client: Client[IO], url: Uri): IO[Either[Throwable, (String, String)]] = {
     val fileName = url.path.toString()
+
     for {
-      request <- Request[IO](Method.GET, url)
+      basicCredentials <- basicCredentialsIO
-        .withHeaders(Authorization(basicCredentials))
+      request = Request[IO](Method.GET, url).withHeaders(Authorization(basicCredentials))
-        .pure[IO]
       result <- client.expect[String](request).redeemWith(
         error => IO(Left(error))
-//          .flatTap(_ => IO.println(s"Request failed to url: $url with error: ${error.getMessage}"))
+        // .flatTap(_ => IO.println(s"Request failed to url: $url with error: ${error.getMessage}")),
         ,
         fileContent => IO(Right((fileName, fileContent)))
-//          .flatTap(_ => IO.println(s"Fetched: $fileName"))
+        // .flatTap(_ => IO.println(s"Fetched: $fileName"))
       )
     } yield result
   }
 
   private def fetchFiles(fileNames: List[String]): IO[List[Either[Throwable, (String, String)]]] = {
-    val urls = fileNames.map(baseUrl / _)
+    val IOUrls = baseUrlIO.map(baseUrl => fileNames.map(baseUrl / _))
     BlazeClientBuilder[IO](global).resource.use { client =>
-      urls.traverse(url => makeRequest(client, url))
+      IOUrls.flatMap(_.traverse(url => makeRequest(client, url)))
     }
   }
 

src/main/scala/fetch/FileNameService.scala

@@ -7,12 +7,6 @@ object FileNameService {
   private val interval = Duration.ofMinutes(30)
   private val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
 
-  // TODO side effect
-  def generateLastNHours(hours: Int, now: LocalDateTime = LocalDateTime.now): List[String] = {
-    val roundNow = roundToInterval(now, false)
-    generate(roundNow.minusHours(hours), roundNow)
-  }
-
   def generateFromDate(date: LocalDate): List[String] = {
     val from = LocalDateTime.of(date.getYear, date.getMonth, date.getDayOfMonth, 0, 0)
     val to = LocalDateTime.of(date.getYear, date.getMonth, date.getDayOfMonth, 23, 59)

src/main/scala/fetch/Main.scala

@@ -9,16 +9,21 @@ import java.time.{LocalDate, LocalDateTime}
 
 object Main {
   def run: IO[Unit] = {
-    val from = LocalDateTime.of(2023, 4, 23, 20, 0)
+    val from = LocalDateTime.of(2023, 4, 28, 10, 0)
-    val to = LocalDateTime.of(2023, 4, 23, 23, 30)
+    val to = LocalDateTime.of(2023, 4, 28, 13, 30)
     for {
-      fetched <- FetchService.fetchInRange(from, to)
+//      fetchResultEither <- FetchService.fetchFromDate(LocalDate.of(2023, 4, 28)).attempt
-//      fetched <- FetchService.fetchFromDate(LocalDate.of(2023, 4, 23))
+      fetchResultEither <- FetchService.fetchInRange(from, to).attempt
-      (errors, successfulDownloads) = fetched.partitionMap(identity)
+      fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList
-      _ <- IO.println(s"Fetch errors: $errors")
+      fetchResult = fetchResultEither.getOrElse(List.empty)
-      saveResults <- successfulDownloads.traverse { case (name, content) => DBService.save(name, content) }
+      (fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
-      _ <- IO.println(s"Save results: $saveResults")
+      saveResults <- successDownloads.traverse { case (name, content) => DBService.save(name, content) }
-    } yield ()
+      (saveErrors, successSaves) = saveResults.partitionMap(identity)
+      successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
+      errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
+      _ <- IO.println(s"errors: $errors")
+      _ <- IO.println(s"successes: $successes")
+    } yield (successes, errors)
   }
 
   def main(args: Array[String]): Unit = {

src/main/scala/server/Server.scala

@@ -36,12 +36,16 @@ object Server extends IOApp {
     // http://localhost:3000/fetch/date/20230423
     case GET -> Root / "fetch" / "date" / ValidDate(date) =>
       val result = for {
-        fetched <- FetchService.fetchFromDate(date)
+        fetchResultEither <- FetchService.fetchFromDate(date).attempt
-        (fetchErrors, successDownloads) = fetched.partitionMap(identity)
+        fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList
-        saved <- successDownloads.traverse { case (name, content) => DBService.save(name, content) }
+        fetchResult = fetchResultEither.getOrElse(List.empty)
-        (saveErrors, successSaves) = saved.partitionMap(identity)
+        (fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
+        saveResults <- successDownloads.traverse { case (name, content) => DBService.save(name, content) }
+        (saveErrors, successSaves) = saveResults.partitionMap(identity)
         successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
-        errors = fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
+        errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
+        _ <- IO.println(s"errors: $errors")
+        _ <- IO.println(s"successes: $successes")
       } yield (successes, errors)
 
       result.flatMap { case (successes, errors) =>