bot/WeatherTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First version of querying data from postgres

Browse Source
This commit is contained in:
Guntis Smaukstelis
2023-12-15 13:46:45 +02:00
parent e89b526a88
commit 3a1f0ac726
9 changed files with 85 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/scala/db/DataService.scala
+4
View File
@@ -6,6 +6,8 @@ import cats.implicits.toTraverseOps
import fetch.FileNameService
import org.typelevel.log4cats.Logger
import org.typelevel.log4cats.slf4j.Slf4jLogger
import parse.Aggregate
import parse.Aggregate.UserQuery
import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
@@ -85,6 +87,8 @@ class DataService private(
def getInRange(from: LocalDateTime, to: LocalDateTime): IO[List[String]] = fileService.getInRange(from, to)
def query(userQuery: UserQuery): IO[Map[String, Option[Aggregate.AggregateValue]]] = postgresService.query(userQuery)
def getDates: IO[List[LocalDate]] = fileService.getDates
def getDatesByMonths(monthList: List[LocalDate]): IO[List[LocalDate]] = {
src/main/scala/db/Main.scala
+9 -2
View File
@@ -1,13 +1,16 @@
package db
import cats.data.NonEmptyList
import cats.effect._
import cats.effect.unsafe.implicits.global
import doobie._
import doobie.implicits._
import doobie.postgres.implicits._
import parse.Aggregate.{AggregateKey, UserQuery}
import java.time.{LocalDate, LocalDateTime}
import java.time.format.DateTimeFormatter
import java.time.temporal.ChronoUnit
object Main {
def transactor[F[_]: Async]: Transactor[F] = Transactor.fromDriverManager[F](
@@ -27,10 +30,14 @@ object Main {
// } yield re
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
val dateTime = LocalDateTime.parse("20231210_1300", formatter)
val from = LocalDateTime.parse("20231212_0000", formatter)
val to = LocalDateTime.parse("20231214_2359", formatter)
val query = UserQuery(NonEmptyList.of("Ainaži", "Rīga", "Kolka", "Vičaki"), "tempAvg", AggregateKey.Max, ChronoUnit.HOURS, from, to)
val result = for {
postgresService <- PostgresService.of(xa)
re <- postgresService.getDateTimeEntries(dateTime)
re <- postgresService.query(query)
} yield re
println(result.unsafeRunSync().toString())
src/main/scala/db/PostgresService.scala
+51 -1
View File
@@ -2,7 +2,7 @@ package db
import cats.data.NonEmptyList
import cats.effect._
import cats.implicits.{catsSyntaxParallelTraverse1, toFoldableOps}
import cats.implicits._
import doobie._
import doobie.implicits._
@@ -11,6 +11,7 @@ import java.time.{LocalDate, LocalDateTime, OffsetDateTime, ZoneId, ZonedDateTim
import doobie.postgres.implicits._
import org.typelevel.log4cats.Logger
import org.typelevel.log4cats.slf4j.Slf4jLogger
import parse.Aggregate.{AggregateKey, AggregateValue, DoubleValue, UserQuery}
import parse.{Parser, WeatherStationData}
object PostgresService {
@@ -34,6 +35,55 @@ class PostgresService(transactor: Transactor[IO], log: Logger[IO]) extends DataS
}
}
/*
SELECT city, AVG(tempmax) AS tempmax -- MAX MIN AVG SUM
FROM weather
WHERE city IN ('Rīga', 'Rēzekne', 'Kolka')
AND dateTime BETWEEN '2023-12-10 00:00:00' AND '2023-12-10 23:59:59'
GROUP BY city;
*/
/*
SELECT city, dateTime, tempmax
FROM weather
WHERE city IN ('Rīga', 'Rēzekne', 'Kolka')
AND dateTime BETWEEN '2023-12-10 00:00:00' AND '2023-12-10 23:59:59'
*/
/*
SELECT city, DATE(dateTime) as day, MAX(tempmax) as max_temp
FROM weather
WHERE city IN ('Rīga', 'Rēzekne', 'Kolka')
AND dateTime BETWEEN '2023-12-01' AND '2023-12-31'
GROUP BY city, DATE(dateTime)
ORDER BY city, day;
*/
def query(userQuery: UserQuery): IO[Map[String, Option[AggregateValue]]] = {
if (List(
AggregateKey.Max,
AggregateKey.Min,
AggregateKey.Avg,
AggregateKey.Sum
).contains(userQuery.key)) {
val query =
(fr"SELECT city, ROUND(CAST(" ++ Fragment.const(userQuery.key.toString.toUpperCase) ++ fr"(" ++ Fragment.const(userQuery.field) ++ fr") AS NUMERIC), 1) AS value FROM weather WHERE " ++
Fragments.in(fr"city", userQuery.cities) ++
fr" AND dateTime BETWEEN ${userQuery.from} AND ${userQuery.to} GROUP BY city")
query.query[(String, Option[Double])]
.to[List]
.transact(transactor)
.map { resultList =>
resultList.map {
case (city, maybeValue) =>
city -> maybeValue.map(DoubleValue)
}.toMap: Map[String, Option[AggregateValue]]
}
} else {
???
}
}
def getDatesByMonths(monthList: List[LocalDate]): IO[List[LocalDate]] = {
val monthYearPairs = monthList.map { localDate =>
val month = localDate.atStartOfDay.atZone(rigaZone).getMonthValue
src/main/scala/parse/Aggregate.scala
+8 -5
View File
@@ -1,5 +1,6 @@
package parse
import cats.data.NonEmptyList
import cats.implicits.{catsSyntaxOptionId, toFoldableOps}
import io.circe.generic.semiauto.deriveEncoder
import io.circe.syntax.EncoderOps
@@ -12,15 +13,17 @@ import java.time.temporal.ChronoUnit
object Aggregate {
final case class UserQuery(
cities: List[String],
field: String,
key: AggregateKey,
granularity: ChronoUnit,
cities: NonEmptyList[String],
field: String,
key: AggregateKey,
granularity: ChronoUnit,
from: LocalDateTime,
to: LocalDateTime,
)
implicit val userQueryEncoder: Encoder[UserQuery] = new Encoder[UserQuery] {
override def apply(userQuery: UserQuery): Json = Json.obj(
"cities" -> Json.fromValues(userQuery.cities.map(Json.fromString)),
"cities" -> Json.fromValues(userQuery.cities.toList.map(Json.fromString)),
"field" -> Json.fromString(userQuery.field),
"key" -> Json.fromString(userQuery.key.toString),
"granularity" -> Json.fromString(userQuery.granularity.toString)
src/main/scala/parse/Main.scala
+2 -1
View File
@@ -1,5 +1,6 @@
package parse
import cats.data.NonEmptyList
import cats.effect.unsafe.implicits.global
import db.FileService
import io.circe.syntax.EncoderOps
@@ -45,7 +46,7 @@ object Main {
val fileService = FileService.of.unsafeRunSync()
val lines = fileService.getInRange(from, to).unsafeRunSync()
val query = UserQuery(List("Ainaži"), "tempAvg", AggregateKey.Avg, ChronoUnit.HOURS)
val query = UserQuery(NonEmptyList.of("Ainaži"), "tempAvg", AggregateKey.Avg, ChronoUnit.HOURS, from, to)
val parsed = Parser.queryData(query, lines)
println(parsed.asJson)
src/main/scala/parse/Parser.scala
+1 -1
View File
@@ -39,7 +39,7 @@ object Parser {
def queryData(userQuery: UserQuery, lines: List[String]): Map[String, Option[AggregateValue]] = {
val weatherByCity = lines
.flatMap(parseLine)
.filter(line => userQuery.cities.contains(line.city))
.filter(line => userQuery.cities.toList.contains(line.city))
.groupBy(_.city)
weatherByCity.map { case (city, weatherStationData) =>
src/main/scala/server/Server.scala
+7 -33
View File
@@ -53,10 +53,14 @@ class Server(dataService: DataService, fetch: FetchService, log: Logger[IO]) {
// http://0.0.0.0:8080/api/query/20230414_2200-20230501_1230/Liepāja,Rēzekne/tempMax/max
case GET -> Root / "query" / DateTimeRange(from, to) / Granularity(granularity) / CityList(cities) / field / AggKey(key) =>
val userQuery = UserQuery(cities, field, key, granularity)
val userQuery = UserQuery(cities, field, key, granularity, from, to)
dataService.getInRange(from, to)
.map(Parser.queryData(userQuery, _))
// dataService.getInRange(from, to)
// .map(Parser.queryData(userQuery, _))
// .map(result => ResponseWrapper(result, userQuery))
// .flatMap(responseWrapper => Ok(responseWrapper.asJson.pretty))
dataService.query(userQuery)
.map(result => ResponseWrapper(result, userQuery))
.flatMap(responseWrapper => Ok(responseWrapper.asJson.pretty))
@@ -84,12 +88,6 @@ class Server(dataService: DataService, fetch: FetchService, log: Logger[IO]) {
).pretty)
}
// http://0.0.0.0:8080/api/show/all_dates
case GET -> Root / "show" / "all_dates" =>
dataService.getDates.flatMap(dates =>
Ok(dates.asJson.pretty)
)
// http://0.0.0.0:8080/api/show/months/202304,202305,202306
case GET -> Root / "show" / "months" / ValidateMonths(monthList) =>
dataService.getDatesByMonths(monthList).flatMap(dates =>
@@ -105,30 +103,6 @@ class Server(dataService: DataService, fetch: FetchService, log: Logger[IO]) {
// http://0.0.0.0:8080/api/show/datetime/20230423_1300
case GET -> Root / "show" / "datetime" / ValidateDateTime(datetime) =>
dataService.getDateTimeEntries(datetime).flatMap(content => Ok(content.asJson))
// http://0.0.0.0:8080/api/show/file/20230423_12:30
case GET -> Root / "show" / "file" / (fileName: String) =>
dataService.readFile(fileName).flatMap(content => Ok(content.asJson))
// http://0.0.0.0:8080/api/getLast24hours
// case GET -> Root / "getLast24hours" => {
// dataService.getLast24Hours.flatMap(content => Ok(content.asJson.pretty))
// }
// http://0.0.0.0:8080/api/help
case GET -> Root / "help" => {
val host = "weather-tool.fly.dev"
Ok(Json.obj(
"aggregate fields" -> WeatherData.getKeys.asJson,
"aggregate keys" -> AggregateKey.getKeys.asJson,
"example urls" -> List(
s"https://$host/api/query/20230414_2200-20230501_1230/Liepāja,Rēzekne/tempMax/max",
s"https://$host/api/fetch/date/20230423",
s"https://$host/api/show/all_dates",
s"https://$host/api/show/date/20230423",
).asJson,
).pretty)
}
}
private val corsConfig = CORSConfig.default
src/main/scala/server/ValidateRoutes.scala
+3 -5
View File
@@ -1,5 +1,6 @@
package server
import cats.data.NonEmptyList
import parse.Aggregate.AggregateKey
import java.time.{LocalDate, LocalDateTime}
@@ -50,11 +51,8 @@ object ValidateRoutes {
}
object CityList {
def unapply(str: String): Option[List[String]] = {
str.split(",").toList match {
case Nil => None
case list => Some(list)
}
def unapply(str: String): Option[NonEmptyList[String]] = {
NonEmptyList.fromList(str.split(",").toList)
}
}