bot/WeatherTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move queries in file manager to postgres

Browse Source
This commit is contained in:
Guntis Smaukstelis
2023-12-12 15:36:30 +02:00
parent 51ed1e2c7a
commit eaf2788031
6 changed files with 74 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/scala/db/DataService.scala
+8 -3
View File
@@ -81,16 +81,21 @@ class DataService private(
def readFile(fileName: String): IO[List[String]] = fileService.readFile(fileName)
def getDateTimeEntries(dateTime: LocalDateTime): IO[List[String]] = postgresService.getDateTimeEntries(dateTime)
def getInRange(from: LocalDateTime, to: LocalDateTime): IO[List[String]] = fileService.getInRange(from, to)
def getDates: IO[List[LocalDate]] = fileService.getDates
def getDatesByMonths(monthList: List[LocalDate]): IO[List[LocalDate]] = {
fileService.getDatesByMonths(monthList)
// postgresService.getDatesByMonths(monthList)
// fileService.getDatesByMonths(monthList)
postgresService.getDatesByMonths(monthList)
}
def getDateFileNames(date: LocalDate): IO[List[String]] = fileService.getDateFileNames(date)
def getDateFileNames(date: LocalDate): IO[List[String]] = {
// fileService.getDateFileNames(date)
postgresService.getDateFileNames(date)
}
// TODO implement getting full data from state
// def getLast24Hours: IO[List[String]] = {
src/main/scala/db/Main.scala
+14 -3
View File
@@ -6,6 +6,9 @@ import doobie._
import doobie.implicits._
import doobie.postgres.implicits._
import java.time.{LocalDate, LocalDateTime}
import java.time.format.DateTimeFormatter
object Main {
def transactor[F[_]: Async]: Transactor[F] = Transactor.fromDriverManager[F](
"org.postgresql.Driver",
@@ -15,13 +18,21 @@ object Main {
)
def main(args: Array[String]): Unit = {
// val xa = transactor[IO]
val xa = transactor[IO]
// val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd")
// val localDate = LocalDate.parse("2023-12-10", formatter)
// val result = for {
// postgresService <- PostgresService.of(xa)
// re <- postgresService.selectWeatherTable
// re <- postgresService.getDateFileNames(localDate)
// } yield re
println(System.getenv("DATABASE_URL"))
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
val dateTime = LocalDateTime.parse("20231210_1300", formatter)
val result = for {
postgresService <- PostgresService.of(xa)
re <- postgresService.getDateTimeEntries(dateTime)
} yield re
println(result.unsafeRunSync().toString())
// val result = createWeatherTable(xa).unsafeRunSync()
// val result = insertInWeatherTable(xa).unsafeRunSync()
src/main/scala/db/PostgresService.scala
+35 -4
View File
@@ -7,7 +7,7 @@ import doobie._
import doobie.implicits._
import java.time.format.DateTimeFormatter
import java.time.{LocalDate, LocalDateTime, ZoneId, ZonedDateTime}
import java.time.{LocalDate, LocalDateTime, OffsetDateTime, ZoneId, ZonedDateTime}
import doobie.postgres.implicits._
import org.typelevel.log4cats.Logger
import org.typelevel.log4cats.slf4j.Slf4jLogger
@@ -64,7 +64,40 @@ class PostgresService(transactor: Transactor[IO], log: Logger[IO]) extends DataS
def getDates: IO[List[LocalDate]] = ???
def getDateFileNames(date: LocalDate): IO[List[String]] = ???
def getDateTimeEntries(dateTime: LocalDateTime): IO[List[String]] = {
fr"""
SELECT datetime || ';' || city || ';' || COALESCE(tempmax::text, '' ) || ';' || COALESCE(tempmin::text, '' ) || ';' || COALESCE(tempavg::text, '' ) || ';' || COALESCE(precipitation::text, '' ) || ';' || COALESCE(windavg::text, '' ) || ';' || COALESCE(windmax::text, '' ) || ';' || COALESCE(tempmax::text, '' ) || ';' || COALESCE(visibilitymin::text, '' ) || ';' || COALESCE(visibilityavg::text, '' ) || ';' || COALESCE(snowavg::text, '' ) || ';' || COALESCE(atmpressure::text, '' ) || ';' || COALESCE(dewpoint::text, '' ) || ';' || COALESCE(humidity::text, '' ) || ';' || COALESCE(sunduration::text, '' ) || ';' || COALESCE(phenomena::text, '' )
FROM weather
WHERE dateTime = $dateTime
ORDER BY city
"""
.query[String]
.to[List]
.transact(transactor)
}
def getDateFileNames(date: LocalDate): IO[List[String]] = {
val year = date.atStartOfDay.atZone(rigaZone).getYear
val month = date.atStartOfDay.atZone(rigaZone).getMonthValue
val day = date.atStartOfDay.atZone(rigaZone).getDayOfMonth
val whereClauses = fr"(EXTRACT(DAY FROM dateTime) = $day AND EXTRACT(MONTH FROM dateTime) = $month AND EXTRACT(YEAR FROM dateTime) = $year)"
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HH00")
(
fr"""
SELECT DISTINCT dateTime
FROM weather
WHERE """ ++ whereClauses ++
fr"ORDER BY dateTime"
)
.query[OffsetDateTime]
.to[List]
.transact(transactor)
.map { dateTimes => {
dateTimes.map(_.format(formatter))
}
}
}
def getResourceContent(path: String): IO[String] = {
val streamResource = Resource.make(IO(getClass.getResourceAsStream(path))) { stream =>
IO(stream.close()).handleErrorWith(_ => IO.unit)
@@ -85,7 +118,6 @@ class PostgresService(transactor: Transactor[IO], log: Logger[IO]) extends DataS
implicit val doubleOptionMeta: Meta[Option[Double]] = Meta[Double].imap(Option(_))(_.getOrElse(Double.NaN))
def insertInWeatherTable(data: List[WeatherStationData]): IO[Int] = {
// IO.blocking {
getResourceContent("/db/insert_weather_table.sql").flatMap { insertTableSql =>
val insertData = data.map(line => {
val zonedTime: ZonedDateTime = line.timestamp.atZone(rigaZone)
@@ -97,7 +129,6 @@ class PostgresService(transactor: Transactor[IO], log: Logger[IO]) extends DataS
.updateMany(insertData)
.transact(transactor)
}
// }.flatten
}
def selectWeatherTable(): IO[List[(String, Option[Double])]] = {
src/main/scala/server/Server.scala
+8 -4
View File
@@ -6,7 +6,7 @@ import com.comcast.ip4s.IpLiteralSyntax
import db.DataService
import fetch.FetchService
import parse.{Aggregate, Parser, WeatherData}
import server.ValidateRoutes.{AggKey, CityList, DateTimeRange, Granularity, ValidDate, ValidateMonths}
import server.ValidateRoutes.{AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateMonths}
import io.circe.{Json, Printer}
import org.http4s._
import org.http4s.dsl.io._
@@ -61,7 +61,7 @@ class Server(dataService: DataService, fetch: FetchService, log: Logger[IO]) {
.flatMap(responseWrapper => Ok(responseWrapper.asJson.pretty))
// http://0.0.0.0:8080/api/fetch/date/20230514
case GET -> Root / "fetch" / "date" / ValidDate(date) =>
case GET -> Root / "fetch" / "date" / ValidateDate(date) =>
val result = for {
fetchResultEither <- fetch.fetchFromDate(date).attempt
fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList
@@ -97,12 +97,16 @@ class Server(dataService: DataService, fetch: FetchService, log: Logger[IO]) {
)
// http://0.0.0.0:8080/api/show/date/20230423
case GET -> Root / "show" / "date" / ValidDate(date) =>
case GET -> Root / "show" / "date" / ValidateDate(date) =>
dataService.getDateFileNames(date).flatMap(fileNames =>
Ok(fileNames.asJson.pretty)
)
// http://0.0.0.0:8080/api/show/file/20230423_12:30.csv
// http://0.0.0.0:8080/api/show/datetime/20230423_1300
case GET -> Root / "show" / "datetime" / ValidateDateTime(datetime) =>
dataService.getDateTimeEntries(datetime).flatMap(content => Ok(content.asJson))
// http://0.0.0.0:8080/api/show/file/20230423_12:30
case GET -> Root / "show" / "file" / (fileName: String) =>
dataService.readFile(fileName).flatMap(content => Ok(content.asJson))
src/main/scala/server/ValidateRoutes.scala
+8 -1
View File
@@ -20,13 +20,20 @@ object ValidateRoutes {
}
}
object ValidDate {
object ValidateDate {
def unapply(str: String): Option[LocalDate] = {
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd")
Try(LocalDate.parse(str, formatter)).toOption
}
}
object ValidateDateTime {
def unapply(str: String): Option[LocalDateTime] = {
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
Try(LocalDateTime.parse(str, formatter)).toOption
}
}
object ValidateMonths {
def unapply(str: String): Option[List[LocalDate]] = {
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd")
web/src/fileManager/FileContent.tsx
+1 -1
View File
@@ -6,7 +6,7 @@ export const FileContent: Component<{getFileName: Accessor<string>}> = (props) =
const fetchFileContent = async (fileName: string) => {
if (fileName === "") return;
await new Promise(resolve => setTimeout(resolve, 500))
const response = await fetch(`${apiHost}/api/show/file/${fileName}`);
const response = await fetch(`${apiHost}/api/show/datetime/${fileName}`);
const text = await response.json();
return text;
}