diff --git a/build.sbt b/build.sbt index b733a56..43786a5 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,6 @@ libraryDependencies ++= Seq( "org.http4s" %% "http4s-blaze-server" % "0.23.9", "org.http4s" %% "http4s-blaze-client" % "0.23.13", "org.http4s" %% "http4s-circe" % http4sVersion, - "org.http4s" %% "http4s-jdk-http-client" % "0.8.0", "ch.qos.logback" % "logback-classic" % "1.2.9", "com.typesafe" % "config" % "1.4.1", diff --git a/src/main/scala/fetch/FetchData.scala b/src/main/scala/fetch/FetchData.scala index 50cafc5..8ce1489 100644 --- a/src/main/scala/fetch/FetchData.scala +++ b/src/main/scala/fetch/FetchData.scala @@ -13,6 +13,7 @@ import java.nio.file.{Files, Paths} import scala.concurrent.ExecutionContext.global object FetchData extends IOApp.Simple { + // TODO wrap config in IO private val config = ConfigFactory.load() private val basicCredentials = BasicCredentials(config.getString("username"), config.getString("password")) private val baseUrl = Uri.unsafeFromString(config.getString("url")) // 20220831_1330.csv @@ -20,6 +21,7 @@ object FetchData extends IOApp.Simple { def saveToFile(fileName: String, content: String): IO[Unit] = { val path = Paths.get(s"data/$fileName") + // TODO redeemWith instead of flatMap IO(Files.writeString(path, content)).attempt.flatMap { case Right(_) => IO(println(s"write: $fileName")) case Left(error) => IO(println(s"Write file '$fileName' failed with error: ${error.getMessage}")) diff --git a/src/main/scala/fetch/FileName.scala b/src/main/scala/fetch/FileName.scala index 3f2d3df..162f5cd 100644 --- a/src/main/scala/fetch/FileName.scala +++ b/src/main/scala/fetch/FileName.scala @@ -8,6 +8,7 @@ object FileName { private val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm") def generateLastHour(): List[String] = { + // TODO IO.timer() IOTimer val now = LocalDateTime.now generate(now.minusHours(1), now) } diff --git a/src/main/scala/parse/MeteoData.scala b/src/main/scala/parse/MeteoData.scala index cd4bd5e..3e87752 100644 --- a/src/main/scala/parse/MeteoData.scala +++ b/src/main/scala/parse/MeteoData.scala @@ -24,9 +24,45 @@ case class MeteoData( atmPressure: Option[Double], dewPoint: Option[Double], airHumidity: Option[Double], - sunshineDuration: Option[Double] + sunshineDuration: Option[Double], ) +sealed trait AggregateMeteo +object AggregateMeteo { + case object tempMax extends AggregateMeteo + + case object tempMin extends AggregateMeteo + + case object tempAvg extends AggregateMeteo + + case object precipitationSum extends AggregateMeteo + + case object windSpeedAvg extends AggregateMeteo + + case object windGustMax extends AggregateMeteo + + case object snowThicknessAvg extends AggregateMeteo + + case object dewPointAvg extends AggregateMeteo + + case object airHumidityAvg extends AggregateMeteo +} + +object Meteo { + def stringToAggregateParam(strParam: String): Option[AggregateMeteo] = strParam match { + case "tempMax" => Some(AggregateMeteo.tempMax) + case "tempMin" => Some(AggregateMeteo.tempMin) + case "tempAvg" => Some(AggregateMeteo.tempAvg) + case "precipitationSum" => Some(AggregateMeteo.precipitationSum) + case "windSpeedAvg" => Some(AggregateMeteo.windSpeedAvg) + case "windGustMax" => Some(AggregateMeteo.windGustMax) + case "snowThicknessAvg" => Some(AggregateMeteo.snowThicknessAvg) + case "dewPointAvg" => Some(AggregateMeteo.dewPointAvg) + case "airHumidityAvg" => Some(AggregateMeteo.airHumidityAvg) + case _ => None + } +} + object MeteoData { def fromDoubles(data: List[Option[Double]]): Option[MeteoData] = data match { case List( diff --git a/src/main/scala/parse/Parser.scala b/src/main/scala/parse/Parser.scala index 92302f2..6b4fcf5 100644 --- a/src/main/scala/parse/Parser.scala +++ b/src/main/scala/parse/Parser.scala @@ -6,6 +6,23 @@ import java.time.format.DateTimeFormatter import scala.io.Source import scala.util.Try +/* +import cats.effect.{IO, Resource} +import java.io.File + +def readFile(file: File): IO[String] = + IO(scala.io.Source.fromFile(file).mkString).handleErrorWith(_ => IO.pure("")) + +def readFiles(dir: File): IO[List[(String, String)]] = + IO(dir.listFiles.toList) + .flatMap(files => + files.traverse { file => + readFile(file).map((file.getName, _)) + } + ) + .handleErrorWith(_ => IO.pure(List.empty)) + */ + object Parser { val data_path = "/Users/guntissmaukstelis/sandbox/hello/data/" @@ -63,6 +80,26 @@ object Parser { def readFromVariable(str: String): List[String] = str.split("\n").toList // Data.csv + def queryData(from: LocalDateTime, to: LocalDateTime, cities: List[String], aggregator: AggregateMeteo): Map[String, Double] = { + val res = getFilesInRange(from, to) + .flatMap(readFromFile) + .flatMap(parseLine) + .filter(line => cities.contains(line.city)) + .groupBy(_.city) + +// res.foreach(println) + + aggregator match { + case AggregateMeteo.tempAvg => res.map { case (city, weatherData) => city -> weatherData.flatMap(_.meteo.tempMax).max } + case AggregateMeteo.tempAvg => res.map { case (city, weatherData) => city -> weatherData.flatMap(_.meteo.tempMin).min } + case AggregateMeteo.tempAvg => res.map { case (city, weatherData) => city -> { + val avgList = weatherData.flatMap(_.meteo.tempAvg) + avgList.sum / avgList.length + }} + case AggregateMeteo.precipitationSum => res.map { case (city, weatherData) => city -> weatherData.flatMap(_.meteo.precipitation).sum } + // TODO add here other aggregateParams + } + } def main(args: Array[String]): Unit = { println("================ start parser") @@ -70,20 +107,9 @@ object Parser { val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm") val start = LocalDateTime.parse("20230409_2200", formatter) val end = LocalDateTime.parse("20230501_1230", formatter) - val files = getFilesInRange(start, end) - val weatherStationData = files - .flatMap(readFromFile) - .flatMap(parseLine) - - // weatherStationData.foreach(println) - - val liepajaWeather = weatherStationData.filter(_.city == "Liepāja") - val maxTempLiepaja = liepajaWeather.flatMap(_.meteo.tempMax).max - val minTempLiepaja = liepajaWeather.flatMap(_.meteo.tempMin).min - val avgTemps = liepajaWeather.flatMap(_.meteo.tempAvg) - val avgTempLiepaja = avgTemps.sum / avgTemps.size - println(s"Liepaja max: $maxTempLiepaja, min: $minTempLiepaja, avg: $avgTempLiepaja") - // liepaja.foreach(println) + val parsed = queryData(start, end, List("Liepāja", "Rēzekne", "randomstr"), AggregateMeteo.tempAvg) +// parsed.foreach(println) + println(parsed.toString()) } } \ No newline at end of file diff --git a/src/main/scala/server/Server.scala b/src/main/scala/server/Server.scala index c1c8ea8..81b024c 100644 --- a/src/main/scala/server/Server.scala +++ b/src/main/scala/server/Server.scala @@ -6,18 +6,19 @@ import org.http4s.dsl.io._ import org.http4s.implicits._ import org.http4s.server.Router import org.http4s.server.blaze.BlazeServerBuilder +import parse.{Meteo, Parser} import java.time.LocalDateTime import java.time.format.DateTimeFormatter import scala.util.Try object Server extends IOApp { - private val formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmm") + private val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm") private val appRoutes = HttpRoutes.of[IO] { - // http://localhost:3000/202304131030-202304132030/riga,liepaja/tempAvg - case GET -> Root / timestampRange / cities / weatherParam => + // http://localhost:3000/20230409_2200-20230501_1230/Liepāja,Rēzekne/tempAvg + case GET -> Root / timestampRange / cities / aggregate => // TODO proly better to Validated with chained errors - val res = for { + val parsedArguments = for { (from, to) <- timestampRange.split("-").toList .map(str => Try(LocalDateTime.parse(str, formatter)).toOption) match { case List(Some(from), Some(to)) => Some(from, to) @@ -27,13 +28,16 @@ object Server extends IOApp { case list => Some(list) case Nil => None } - param <- if (weatherParam.isEmpty) None else Some(weatherParam) - } - // TODO yield to parser method which accepts those args - yield s"Timestamp From: $from, Timestamp To: $to, Cities: ${cityList.mkString(", ")}, Weather Param: $param" + aggregate <- Meteo.stringToAggregateParam(aggregate) - res match { - case Some(responseTxt) => Ok(responseTxt) + } + yield (from, to, cityList, aggregate) + + parsedArguments match { + case Some((from, to, cityList, aggregate)) => { + val resultData = Parser.queryData(from, to, cityList, aggregate) + Ok(resultData.toString()) + } case _ => BadRequest(s"Invalid request format") } }