From 7e3d58bec07959d452675947df4c90f0f5237b89 Mon Sep 17 00:00:00 2001 From: Guntis Smaukstelis Date: Mon, 1 May 2023 16:43:21 +0300 Subject: [PATCH] Add conf into IO and handle IO throwable --- src/main/scala/fetch/FetchService.scala | 28 ++++++++++++++-------- src/main/scala/fetch/FileNameService.scala | 6 ----- src/main/scala/fetch/Main.scala | 23 +++++++++++------- src/main/scala/server/Server.scala | 14 +++++++---- 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/src/main/scala/fetch/FetchService.scala b/src/main/scala/fetch/FetchService.scala index 896984c..1876d86 100644 --- a/src/main/scala/fetch/FetchService.scala +++ b/src/main/scala/fetch/FetchService.scala @@ -12,31 +12,39 @@ import java.time.{LocalDate, LocalDateTime} import scala.concurrent.ExecutionContext.global object FetchService { - // TODO wrap config in IO private val config = ConfigFactory.load() - private val basicCredentials = BasicCredentials(config.getString("username"), config.getString("password")) - private val baseUrl = Uri.unsafeFromString(config.getString("url")) // 20220831_1330.csv + private def getIOString(path: String): IO[String] = + if (config.hasPath(path)) IO.pure(config.getString(path)) + else IO.raiseError(new RuntimeException(s"Missing configuration: $path")) + + private val basicCredentialsIO: IO[BasicCredentials] = + for { + username <- getIOString("username") + password <- getIOString("password") + } yield BasicCredentials(username, password) + + private val baseUrlIO: IO[Uri] = getIOString("url").map(Uri.unsafeFromString) // 20220831_1330.csv private def makeRequest(client: Client[IO], url: Uri): IO[Either[Throwable, (String, String)]] = { val fileName = url.path.toString() + for { - request <- Request[IO](Method.GET, url) - .withHeaders(Authorization(basicCredentials)) - .pure[IO] + basicCredentials <- basicCredentialsIO + request = Request[IO](Method.GET, url).withHeaders(Authorization(basicCredentials)) result <- client.expect[String](request).redeemWith( error => IO(Left(error)) -// .flatTap(_ => IO.println(s"Request failed to url: $url with error: ${error.getMessage}")) + // .flatTap(_ => IO.println(s"Request failed to url: $url with error: ${error.getMessage}")), , fileContent => IO(Right((fileName, fileContent))) -// .flatTap(_ => IO.println(s"Fetched: $fileName")) + // .flatTap(_ => IO.println(s"Fetched: $fileName")) ) } yield result } private def fetchFiles(fileNames: List[String]): IO[List[Either[Throwable, (String, String)]]] = { - val urls = fileNames.map(baseUrl / _) + val IOUrls = baseUrlIO.map(baseUrl => fileNames.map(baseUrl / _)) BlazeClientBuilder[IO](global).resource.use { client => - urls.traverse(url => makeRequest(client, url)) + IOUrls.flatMap(_.traverse(url => makeRequest(client, url))) } } diff --git a/src/main/scala/fetch/FileNameService.scala b/src/main/scala/fetch/FileNameService.scala index 30c48c1..b24de6d 100644 --- a/src/main/scala/fetch/FileNameService.scala +++ b/src/main/scala/fetch/FileNameService.scala @@ -7,12 +7,6 @@ object FileNameService { private val interval = Duration.ofMinutes(30) private val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm") - // TODO side effect - def generateLastNHours(hours: Int, now: LocalDateTime = LocalDateTime.now): List[String] = { - val roundNow = roundToInterval(now, false) - generate(roundNow.minusHours(hours), roundNow) - } - def generateFromDate(date: LocalDate): List[String] = { val from = LocalDateTime.of(date.getYear, date.getMonth, date.getDayOfMonth, 0, 0) val to = LocalDateTime.of(date.getYear, date.getMonth, date.getDayOfMonth, 23, 59) diff --git a/src/main/scala/fetch/Main.scala b/src/main/scala/fetch/Main.scala index 6a2b127..e7aeb0f 100644 --- a/src/main/scala/fetch/Main.scala +++ b/src/main/scala/fetch/Main.scala @@ -9,16 +9,21 @@ import java.time.{LocalDate, LocalDateTime} object Main { def run: IO[Unit] = { - val from = LocalDateTime.of(2023, 4, 23, 20, 0) - val to = LocalDateTime.of(2023, 4, 23, 23, 30) + val from = LocalDateTime.of(2023, 4, 28, 10, 0) + val to = LocalDateTime.of(2023, 4, 28, 13, 30) for { - fetched <- FetchService.fetchInRange(from, to) -// fetched <- FetchService.fetchFromDate(LocalDate.of(2023, 4, 23)) - (errors, successfulDownloads) = fetched.partitionMap(identity) - _ <- IO.println(s"Fetch errors: $errors") - saveResults <- successfulDownloads.traverse { case (name, content) => DBService.save(name, content) } - _ <- IO.println(s"Save results: $saveResults") - } yield () +// fetchResultEither <- FetchService.fetchFromDate(LocalDate.of(2023, 4, 28)).attempt + fetchResultEither <- FetchService.fetchInRange(from, to).attempt + fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList + fetchResult = fetchResultEither.getOrElse(List.empty) + (fetchErrors, successDownloads) = fetchResult.partitionMap(identity) + saveResults <- successDownloads.traverse { case (name, content) => DBService.save(name, content) } + (saveErrors, successSaves) = saveResults.partitionMap(identity) + successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s") + errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}") + _ <- IO.println(s"errors: $errors") + _ <- IO.println(s"successes: $successes") + } yield (successes, errors) } def main(args: Array[String]): Unit = { diff --git a/src/main/scala/server/Server.scala b/src/main/scala/server/Server.scala index 5d788b4..5807d1e 100644 --- a/src/main/scala/server/Server.scala +++ b/src/main/scala/server/Server.scala @@ -36,12 +36,16 @@ object Server extends IOApp { // http://localhost:3000/fetch/date/20230423 case GET -> Root / "fetch" / "date" / ValidDate(date) => val result = for { - fetched <- FetchService.fetchFromDate(date) - (fetchErrors, successDownloads) = fetched.partitionMap(identity) - saved <- successDownloads.traverse { case (name, content) => DBService.save(name, content) } - (saveErrors, successSaves) = saved.partitionMap(identity) + fetchResultEither <- FetchService.fetchFromDate(date).attempt + fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList + fetchResult = fetchResultEither.getOrElse(List.empty) + (fetchErrors, successDownloads) = fetchResult.partitionMap(identity) + saveResults <- successDownloads.traverse { case (name, content) => DBService.save(name, content) } + (saveErrors, successSaves) = saveResults.partitionMap(identity) successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s") - errors = fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}") + errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}") + _ <- IO.println(s"errors: $errors") + _ <- IO.println(s"successes: $successes") } yield (successes, errors) result.flatMap { case (successes, errors) =>