diff --git a/src/main/scala/db/DBService.scala b/src/main/scala/db/DBService.scala new file mode 100644 index 0000000..b099775 --- /dev/null +++ b/src/main/scala/db/DBService.scala @@ -0,0 +1,74 @@ +package db + +import cats.effect.unsafe.implicits.global +import cats.effect.{IO, Resource} +import cats.implicits.toTraverseOps + +import java.io.File +import java.nio.file.{Files, Paths} +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import scala.io.Source +import scala.util.Try + +object DBService { + private val dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm") + private val dataPath = "/Users/guntissmaukstelis/sandbox/WeatherTool/data/" + + private def readFile(fileName: String): IO[String] = { + val file = new File(dataPath, fileName) + val sourceResource = Resource.fromAutoCloseable(IO(Source.fromFile(file))) + sourceResource.use(source => IO(source.mkString)).handleErrorWith(_ => IO.pure("")) + } + + private def readFileNames(path: String): IO[List[String]] = + IO(new File(path).listFiles.toList.map(_.getName)) + .handleErrorWith(_ => IO.pure(List.empty)) + + private def inRange(fileName: String, from: LocalDateTime, to: LocalDateTime): Boolean = { + def fileToDateTime(fileName: String): Option[LocalDateTime] = { + val dateString = fileName.split("\\.").head + Try(LocalDateTime.parse(dateString, dateFormatter)).toOption + } + + val fileDateTime = fileToDateTime(fileName.stripSuffix (".csv")) + fileDateTime match { + case Some (date) => date.plusSeconds (1).isAfter (from) && date.minusSeconds (1).isBefore (to) + case None => false + } + } + + def getInRange(from: LocalDateTime, to: LocalDateTime): IO[List[String]] = { + for { + fileNames <- readFileNames(dataPath) + .map (_.filter (inRange (_, from, to))) + fileContent <- fileNames.traverse(readFile) + } yield fileContent + } + + def save(fileName: String, content: String): IO[Unit] = { + val path = Paths.get(s"$dataPath/$fileName") + // TODO redeemWith instead of flatMap + IO(Files.writeString(path, content)).attempt.flatMap { + case Right(_) => IO.println(s"write: $fileName") + case Left(error) => IO.println(s"Write file '$fileName' failed with error: ${error.getMessage}") + } + } + + // TODO remove this. Just testing + private def run: IO[Unit] = { + println("----------------> db main") + + val from = LocalDateTime.parse("20230414_2200", dateFormatter) + val to = LocalDateTime.parse("20230501_1230", dateFormatter) + + for { + lines <- getInRange(from, to) + _ <- lines.traverse(IO.println) + } yield () + } + + def main(args: Array[String]): Unit = { + run.unsafeRunSync() + } +} diff --git a/src/main/scala/fetch/FetchData.scala b/src/main/scala/fetch/FetchData.scala index 8ce1489..7503a88 100644 --- a/src/main/scala/fetch/FetchData.scala +++ b/src/main/scala/fetch/FetchData.scala @@ -17,16 +17,6 @@ object FetchData extends IOApp.Simple { private val config = ConfigFactory.load() private val basicCredentials = BasicCredentials(config.getString("username"), config.getString("password")) private val baseUrl = Uri.unsafeFromString(config.getString("url")) // 20220831_1330.csv -// private val baseUrl = Uri.unsafeFromString("https://jsonplaceholder.typicode.com/") // todos/1 - - def saveToFile(fileName: String, content: String): IO[Unit] = { - val path = Paths.get(s"data/$fileName") - // TODO redeemWith instead of flatMap - IO(Files.writeString(path, content)).attempt.flatMap { - case Right(_) => IO(println(s"write: $fileName")) - case Left(error) => IO(println(s"Write file '$fileName' failed with error: ${error.getMessage}")) - } - } def makeRequest(client: Client[IO], url: Uri): IO[Unit] = { val fileName = url.path.toString() @@ -36,7 +26,7 @@ object FetchData extends IOApp.Simple { .pure[IO] responseOrError <- client.expect[String](request).attempt _ <- responseOrError match { - case Right(response) => saveToFile(fileName, response) + case Right(response) => db.DBService.save(fileName, response) case Left(error) => IO(println(s"Request failed to url: $url with error: ${error.getMessage}")) } } yield () @@ -44,7 +34,6 @@ object FetchData extends IOApp.Simple { def run: IO[Unit] = { val fileNames = FileName.generateLastNHours(10) - // fileNames.foreach(println) val urls = fileNames.map(baseUrl / _) BlazeClientBuilder[IO](global).resource.use { client => urls.traverse(url => makeRequest(client, url)) // urls.map(...).sequence diff --git a/src/main/scala/parse/Parser.scala b/src/main/scala/parse/Parser.scala index 6b4fcf5..79bf3b0 100644 --- a/src/main/scala/parse/Parser.scala +++ b/src/main/scala/parse/Parser.scala @@ -6,23 +6,6 @@ import java.time.format.DateTimeFormatter import scala.io.Source import scala.util.Try -/* -import cats.effect.{IO, Resource} -import java.io.File - -def readFile(file: File): IO[String] = - IO(scala.io.Source.fromFile(file).mkString).handleErrorWith(_ => IO.pure("")) - -def readFiles(dir: File): IO[List[(String, String)]] = - IO(dir.listFiles.toList) - .flatMap(files => - files.traverse { file => - readFile(file).map((file.getName, _)) - } - ) - .handleErrorWith(_ => IO.pure(List.empty)) - */ - object Parser { val data_path = "/Users/guntissmaukstelis/sandbox/hello/data/" diff --git a/src/main/scala/server/Server.scala b/src/main/scala/server/Server.scala index 81b024c..9f022c3 100644 --- a/src/main/scala/server/Server.scala +++ b/src/main/scala/server/Server.scala @@ -15,8 +15,8 @@ import scala.util.Try object Server extends IOApp { private val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm") private val appRoutes = HttpRoutes.of[IO] { - // http://localhost:3000/20230409_2200-20230501_1230/Liepāja,Rēzekne/tempAvg - case GET -> Root / timestampRange / cities / aggregate => + // http://localhost:3000/query/20230409_2200-20230501_1230/Liepāja,Rēzekne/tempAvg + case GET -> Root / "query" / timestampRange / cities / aggregate => // TODO proly better to Validated with chained errors val parsedArguments = for { (from, to) <- timestampRange.split("-").toList @@ -40,6 +40,16 @@ object Server extends IOApp { } case _ => BadRequest(s"Invalid request format") } + + case GET -> Root / "fetch" / dateRange => + ??? + + case GET -> Root / "show" / "fetched_dates" => + ??? + + case GET -> Root / "show" / dateRange => + ??? + } private val httpApp = Router("/" -> appRoutes).orNotFound