diff --git a/.jvmopts b/.jvmopts new file mode 100644 index 0000000..bd71177 --- /dev/null +++ b/.jvmopts @@ -0,0 +1,2 @@ +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED \ No newline at end of file diff --git a/project/assembly.sbt b/project/assembly.sbt index 72477a2..9ae005c 100644 --- a/project/assembly.sbt +++ b/project/assembly.sbt @@ -1 +1,2 @@ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0") +addSbtPlugin("nl.gn0s1s" % "sbt-dotenv" % "3.0.0") diff --git a/src/main/scala/db/DBConnection.scala b/src/main/scala/db/DBConnection.scala index cb36d93..59c6094 100644 --- a/src/main/scala/db/DBConnection.scala +++ b/src/main/scala/db/DBConnection.scala @@ -14,7 +14,7 @@ object DBConnection { private def getDbPassword = sys.env.getOrElse("POSTGRES_PASSWORD", "") private def getDatabaseUrl: String = - sys.env.getOrElse("DATABASE_URL", s"postgres://${getDbUser}:${getDbPassword}@postgres:5432/${getDbName}") + sys.env.getOrElse("DATABASE_URL", s"postgres://${getDbUser}:${getDbPassword}@0.0.0.0:5432/${getDbName}") // host - postgres for docker, 0.0.0.0 outside docker private def parseUrl(url: String): Either[String, PostgresConfig] = { Try { diff --git a/src/main/scala/db/PostgresService.scala b/src/main/scala/db/PostgresService.scala index 3362657..b8c53c7 100644 --- a/src/main/scala/db/PostgresService.scala +++ b/src/main/scala/db/PostgresService.scala @@ -7,7 +7,7 @@ import doobie._ import doobie.implicits._ import java.time.format.DateTimeFormatter -import java.time.{LocalDate, LocalDateTime, OffsetDateTime, ZoneId, ZonedDateTime} +import java.time.{LocalDate, LocalDateTime, OffsetDateTime} import doobie.postgres.implicits._ import org.typelevel.log4cats.Logger import org.typelevel.log4cats.slf4j.Slf4jLogger @@ -23,8 +23,6 @@ object PostgresService { } class PostgresService(transactor: Transactor[IO], log: Logger[IO]) { - // in pgAdmin run: ```SET TIMEZONE = 'Europe/Riga';``` - def save(fileName: String, content: String): IO[String] = { val strLines = content.split(System.lineSeparator()).toList val weatherStationData = strLines.flatMap(Parser.parseLine) @@ -36,6 +34,30 @@ class PostgresService(transactor: Transactor[IO], log: Logger[IO]) { } } + def queryCountry(from: LocalDateTime, to: LocalDateTime, fieldList: NonEmptyList[String]): IO[Map[String, (Option[Double], Option[Double], Option[Double], Option[Double])]] = { + def queryForField(field: String, from: LocalDateTime, to: LocalDateTime): ConnectionIO[(String, (Option[Double], Option[Double], Option[Double], Option[Double]))] = + { + val query = + fr"SELECT " ++ + fr" ROUND(CAST(MIN(" ++ Fragment.const(field) ++ fr") AS NUMERIC), 1), " ++ + fr" ROUND(CAST(MAX(" ++ Fragment.const(field) ++ fr") AS NUMERIC), 1), " ++ + fr" ROUND(CAST(AVG(" ++ Fragment.const(field) ++ fr") AS NUMERIC), 1), " ++ + fr" ROUND(CAST(SUM(" ++ Fragment.const(field) ++ fr") AS NUMERIC), 1)" ++ + fr" FROM weather" ++ + fr" WHERE dateTime BETWEEN $from AND $to" + + query.query[(Option[Double], Option[Double], Option[Double], Option[Double])] + .unique + .map { case (min, max, avg, sum) => + field -> (min, max, avg, sum) + } + } + + fieldList.toList.traverse { field => + queryForField(field, from, to).transact(transactor) + }.map(_.toMap) + } + def query(userQuery: UserQuery): IO[Map[String, Option[AggregateValue]]] = { if (userQuery.field == "phenomena") { // this handles strings // TODO query list and distinct values from phenomena @@ -171,12 +193,6 @@ class PostgresService(transactor: Transactor[IO], log: Logger[IO]) { .transact(transactor) } - def readFile(fileName: String): IO[List[String]] = ??? - - def getInRange(from: LocalDateTime, to: LocalDateTime): IO[List[String]] = ??? - - def getDates: IO[List[LocalDate]] = ??? - def getDateTimeEntries(dateTime: LocalDateTime): IO[List[String]] = { val columnNames = List("City; TempMax; TempMin; TempAvg; Precipitation; WindAvg; WindMax; VisibilityMin; VisibilityAvg; SnowAvg; AtmPressure; DewPoint; Humidity; SunDuration; Phenomena") val result = fr""" @@ -247,38 +263,4 @@ class PostgresService(transactor: Transactor[IO], log: Logger[IO]) { .transact(transactor) } } - - def selectWeatherTable(): IO[List[(String, Option[Double])]] = { - val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm") - val from = LocalDateTime.parse("20230516_0100", formatter) - val to = LocalDateTime.parse("20230516_1500", formatter) - val citiesNel = NonEmptyList.of("Rīga", "Rēzekne") - val granularity = "HOUR" // "HOUR", "DAY", "MONTH", "YEAR" - val columnName = "tempMin" - - val baseQuery = - fr""" - SELECT - city, - MIN(""" ++ Fragment.const(columnName) ++ fr""") AS tempMin - FROM weather - WHERE - """ ++ Fragments.in(fr"city", citiesNel) ++ fr""" - AND dateTime BETWEEN $from AND $to - AND """ ++ Fragment.const(columnName) ++fr""" IS NOT NULL - GROUP BY city, EXTRACT(""" ++ Fragment.const(granularity) ++ fr""" FROM dateTime) - """ - - baseQuery - .query[(String, Option[Double])] - .to[List] - .transact(transactor) - } - - def dropWeatherTable(): IO[Int] = { - for { - dropTableSql <- getResourceContent("/db/drop_weather_table.sql") - result <- Update0(dropTableSql, None).run.transact(transactor) - } yield result - } } diff --git a/src/main/scala/server/Server.scala b/src/main/scala/server/Server.scala index 4191410..9f9f250 100644 --- a/src/main/scala/server/Server.scala +++ b/src/main/scala/server/Server.scala @@ -5,8 +5,8 @@ import cats.implicits.toTraverseOps import com.comcast.ip4s.IpLiteralSyntax import db.PostgresService import fetch.FetchService -import parse.{Aggregate} -import server.ValidateRoutes.{AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateMonths} +import parse.Aggregate +import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateMonths} import io.circe.{Json, Printer} import org.http4s._ import org.http4s.dsl.io._ @@ -50,14 +50,19 @@ class Server(postgresService: PostgresService, fetch: FetchService, log: Logger[ private val apiRoutes = HttpRoutes.of[IO] { - // http://0.0.0.0:8080/api/query/20230414_2200-20230501_1230/Liepāja,Rēzekne/tempMax/max - case GET -> Root / "query" / DateTimeRange(from, to) / Granularity(granularity) / CityList(cities) / field / AggKey(key) => + // http://0.0.0.0:8080/api/query/city/Liepāja,Rēzekne/20230414_2200-20230501_1230/hour/tempMax/max + case GET -> Root / "query" / "city" / CityList(cities) / DateTimeRange(from, to) / Granularity(granularity) / field / AggKey(key) => val userQuery = UserQuery(cities, field, key, granularity, from, to) postgresService.query(userQuery) .map(result => ResponseWrapper(result, userQuery)) .flatMap(responseWrapper => Ok(responseWrapper.asJson.pretty)) + // http://0.0.0.0:8080/api/query/country/20230414_2200-20230501_1230/tempMax,tempMin,tempAvg,precipitation + case GET -> Root / "query" / "country" / DateTimeRange(from, to) / AggFieldList(fieldList) => + postgresService.queryCountry(from, to, fieldList) + .flatMap(result => Ok(result.asJson.pretty)) + // http://0.0.0.0:8080/api/fetch/date/20230514 case GET -> Root / "fetch" / "date" / ValidateDate(date) => val result = for { diff --git a/src/main/scala/server/ValidateRoutes.scala b/src/main/scala/server/ValidateRoutes.scala index 4824fd1..a8de66c 100644 --- a/src/main/scala/server/ValidateRoutes.scala +++ b/src/main/scala/server/ValidateRoutes.scala @@ -2,6 +2,7 @@ package server import cats.data.NonEmptyList import parse.Aggregate.AggregateKey +import parse.WeatherData import java.time.{LocalDate, LocalDateTime} import java.time.format.DateTimeFormatter @@ -56,6 +57,14 @@ object ValidateRoutes { } } + object AggFieldList { + def unapply(str: String): Option[NonEmptyList[String]] = { + val weatherFields = WeatherData.getKeys + val filteredList = str.split(",").toList.filter(weatherFields.contains) + NonEmptyList.fromList(filteredList) + } + } + object AggKey { def unapply(str: String): Option[AggregateKey] = { AggregateKey.fromString(str) diff --git a/web/src/App.tsx b/web/src/App.tsx index 466f891..bd96d31 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,31 +1,34 @@ import { Component, createSignal } from 'solid-js'; -import { Aggregator } from './aggregator/Aggregator'; import styles from './css/App.module.css'; -import { FileManager } from './fileManager/FileManager'; +import { Country } from './country/Country'; +import { Cities } from './cities/Cities'; +import { Database } from './database/Database'; console.log("env:", import.meta.env.MODE); console.log("api host:", import.meta.env.VITE_API_HOST); -type Section = "aggregator" | "mapView" | "fileManager"; +type Section = "cities" | "country" | "database"; const App: Component = () => { - const [getSection, setSection] = createSignal
("aggregator"); + const [getSection, setSection] = createSignal
("country"); const section = () => { switch(getSection()) { - case "fileManager": return ; - case "aggregator": + case "database": return ; + case "country": return ; + case "cities": return default: - return ; + return ; } } return (
- setSection("aggregator")}>aggregator | - setSection("fileManager")}>file manager + setSection("cities")}>cities | + setSection("country")}>latvia | + setSection("database")}>database
{ section() }
diff --git a/web/src/aggregator/Aggregator.tsx b/web/src/cities/Cities.tsx similarity index 94% rename from web/src/aggregator/Aggregator.tsx rename to web/src/cities/Cities.tsx index bc8e3f6..cb78ccd 100644 --- a/web/src/aggregator/Aggregator.tsx +++ b/web/src/cities/Cities.tsx @@ -6,12 +6,12 @@ import { SelectCity } from "./SelectCity"; import { SelectField } from "./SelectField"; import { SelectGranularity } from "./SelectGranularity"; import { SelectKey } from "./SelectKey"; -import { SelectTimeRange } from "./SelectTimeRange"; +import { SelectTimeRange } from "../components/SelectTimeRange"; const nowRounded = new Date(new Date().setMinutes(30)); const dayAgo = moment(nowRounded).subtract(1, "days").subtract(30, "minutes").toDate(); -export function Aggregator() { +export function Cities() { const [getCities, setCities] = createSignal>(new Set([])); const [getStart, setStart] = createSignal(dayAgo); const [getEnd, setEnd] = createSignal(nowRounded); @@ -21,7 +21,7 @@ export function Aggregator() { return (
-

Aggregator

+

Cities

diff --git a/web/src/aggregator/SelectCity.tsx b/web/src/cities/SelectCity.tsx similarity index 100% rename from web/src/aggregator/SelectCity.tsx rename to web/src/cities/SelectCity.tsx diff --git a/web/src/aggregator/SelectField.tsx b/web/src/cities/SelectField.tsx similarity index 100% rename from web/src/aggregator/SelectField.tsx rename to web/src/cities/SelectField.tsx diff --git a/web/src/aggregator/SelectGranularity.tsx b/web/src/cities/SelectGranularity.tsx similarity index 100% rename from web/src/aggregator/SelectGranularity.tsx rename to web/src/cities/SelectGranularity.tsx diff --git a/web/src/aggregator/SelectKey.tsx b/web/src/cities/SelectKey.tsx similarity index 100% rename from web/src/aggregator/SelectKey.tsx rename to web/src/cities/SelectKey.tsx diff --git a/web/src/aggregator/SelectOrder.tsx b/web/src/cities/SelectOrder.tsx similarity index 100% rename from web/src/aggregator/SelectOrder.tsx rename to web/src/cities/SelectOrder.tsx diff --git a/web/src/aggregator/chart/CityChart.tsx b/web/src/cities/chart/CityChart.tsx similarity index 100% rename from web/src/aggregator/chart/CityChart.tsx rename to web/src/cities/chart/CityChart.tsx diff --git a/web/src/aggregator/chart/CityLargeChart.tsx b/web/src/cities/chart/CityLargeChart.tsx similarity index 100% rename from web/src/aggregator/chart/CityLargeChart.tsx rename to web/src/cities/chart/CityLargeChart.tsx diff --git a/web/src/aggregator/chart/CityResult.tsx b/web/src/cities/chart/CityResult.tsx similarity index 100% rename from web/src/aggregator/chart/CityResult.tsx rename to web/src/cities/chart/CityResult.tsx diff --git a/web/src/aggregator/chart/CustomChart.ts b/web/src/cities/chart/CustomChart.ts similarity index 100% rename from web/src/aggregator/chart/CustomChart.ts rename to web/src/cities/chart/CustomChart.ts diff --git a/web/src/aggregator/helpers.ts b/web/src/cities/helpers.ts similarity index 100% rename from web/src/aggregator/helpers.ts rename to web/src/cities/helpers.ts diff --git a/web/src/aggregator/map/MapView.tsx b/web/src/cities/map/MapView.tsx similarity index 100% rename from web/src/aggregator/map/MapView.tsx rename to web/src/cities/map/MapView.tsx diff --git a/web/src/aggregator/map/WindInputs.tsx b/web/src/cities/map/WindInputs.tsx similarity index 100% rename from web/src/aggregator/map/WindInputs.tsx rename to web/src/cities/map/WindInputs.tsx diff --git a/web/src/aggregator/map/cityCoords.ts b/web/src/cities/map/cityCoords.ts similarity index 100% rename from web/src/aggregator/map/cityCoords.ts rename to web/src/cities/map/cityCoords.ts diff --git a/web/src/aggregator/map/windAngles.ts b/web/src/cities/map/windAngles.ts similarity index 100% rename from web/src/aggregator/map/windAngles.ts rename to web/src/cities/map/windAngles.ts diff --git a/web/src/aggregator/result/QueryResult.tsx b/web/src/cities/result/QueryResult.tsx similarity index 92% rename from web/src/aggregator/result/QueryResult.tsx rename to web/src/cities/result/QueryResult.tsx index 2144a7a..b0dbaae 100644 --- a/web/src/aggregator/result/QueryResult.tsx +++ b/web/src/cities/result/QueryResult.tsx @@ -22,7 +22,7 @@ export const QueryResult: Component<{ const fetchQuery = async (timestamp: number) => { if (cities() === "") return new Error("ERROR: Select cities!"); await new Promise(resolve => setTimeout(resolve, 500)) - const response = await fetch(`${apiHost}/api/query/${queryStart()}-${queryEnd()}/${props.getGranularity()}/${cities()}/${props.getField()}/${props.getKey()}`); + const response = await fetch(`${apiHost}/api/query/city/${cities()}/${queryStart()}-${queryEnd()}/${props.getGranularity()}/${props.getField()}/${props.getKey()}`); const json = await response.json(); return json; } diff --git a/web/src/aggregator/result/Result.tsx b/web/src/cities/result/Result.tsx similarity index 100% rename from web/src/aggregator/result/Result.tsx rename to web/src/cities/result/Result.tsx diff --git a/web/src/aggregator/SelectTimeRange.tsx b/web/src/components/SelectTimeRange.tsx similarity index 100% rename from web/src/aggregator/SelectTimeRange.tsx rename to web/src/components/SelectTimeRange.tsx diff --git a/web/src/country/Country.tsx b/web/src/country/Country.tsx new file mode 100644 index 0000000..83636bd --- /dev/null +++ b/web/src/country/Country.tsx @@ -0,0 +1,42 @@ +import { createSignal } from "solid-js"; + +import { QueryResult } from "./result/QueryResult"; +import { SelectTimeRange } from "../components/SelectTimeRange"; +import moment from "moment"; +import { weatherField } from "../consts"; + +const nowRounded = new Date(new Date().setMinutes(30)); +const dayAgo = moment(nowRounded).subtract(1, "days").subtract(30, "minutes").toDate(); + +export function Country() { + const weatherFieldNumeric = weatherField.filter(f => f !== "phenomena"); + const [getStart, setStart] = createSignal(dayAgo); + const [getEnd, setEnd] = createSignal(nowRounded); + const [getFields, setFields] = createSignal(weatherFieldNumeric); + + return ( +
+

Latvia

+
+
+ column1 +
+
+ +
+
+ +
+
+
+ ); +} \ No newline at end of file diff --git a/web/src/country/result/QueryResult.tsx b/web/src/country/result/QueryResult.tsx new file mode 100644 index 0000000..4d9f3e2 --- /dev/null +++ b/web/src/country/result/QueryResult.tsx @@ -0,0 +1,52 @@ +import moment from "moment"; +import { Accessor, Component, createResource, createSignal } from "solid-js"; + +import { apiHost } from "../../consts"; +import { Result } from "./Result"; + + +export const QueryResult: Component<{ + getStart: Accessor, + getEnd: Accessor, + getFields: Accessor, +}> = (props) => { + const queryStart = () => moment(props.getStart()).format("YYYYMMDD_HHmm"); + const queryEnd = () => moment(props.getEnd()).format("YYYYMMDD_HHmm"); + const [getTimestamp, setTimestamp] = createSignal(0); + + const fetchQuery = async (timestamp: number) => { + if (props.getFields().length === 0) return new Error("ERROR: Select weather parameters!"); + await new Promise(resolve => setTimeout(resolve, 500)) + const response = await fetch(`${apiHost}/api/query/country/${queryStart()}-${queryEnd()}/${props.getFields().join(",")}`); + const json = await response.json(); + return json; + } + + const [queryResource] = createResource(getTimestamp, fetchQuery); + + return ( +
+ setTimestamp(Date.now())} + /> + { queryResource.loading && ( +
+ + Loading query +
+ )} + { queryResource.error && ( +
Error while querying: ${queryResource.error}
+ )} + { queryResource() && queryResource() instanceof Error && +
{ queryResource().message }
+ } + { queryResource() && + + } +
+ ); +} \ No newline at end of file diff --git a/web/src/country/result/Result.tsx b/web/src/country/result/Result.tsx new file mode 100644 index 0000000..dccbbb5 --- /dev/null +++ b/web/src/country/result/Result.tsx @@ -0,0 +1,34 @@ +import { Component, Resource } from "solid-js"; + +export interface ResultData { + [field: string]: [number, number, number, number]; +} + +export const Result: Component<{ + result: Resource +}> = ({ result: resultResource }) => { + const countryData = () => Object.entries(resultResource()!) + .sort((a, b) => a[0] > b[0] ? 1 : -1) + .map(([field, values]) => + {field} + {values[0]} + {values[1]} + {values[2]} + {values[3]} + ) + + return ( +
+ + + + + + + + + { countryData() } +
Paramminmaxavgsum
+
+ ); +} \ No newline at end of file diff --git a/web/src/fileManager/Calendar.tsx b/web/src/database/Calendar.tsx similarity index 100% rename from web/src/fileManager/Calendar.tsx rename to web/src/database/Calendar.tsx diff --git a/web/src/fileManager/FileManager.tsx b/web/src/database/Database.tsx similarity index 94% rename from web/src/fileManager/FileManager.tsx rename to web/src/database/Database.tsx index abcf344..0562ab6 100644 --- a/web/src/fileManager/FileManager.tsx +++ b/web/src/database/Database.tsx @@ -7,7 +7,7 @@ import { FetchFiles } from "./FetchFiles"; import { FileContent } from "./FileContent"; import { FileNameList } from "./FileNameList"; -export function FileManager() { +export function Database() { const [getDate, setDate] = createSignal(new Date()); const [getFileName, setFileName] = createSignal(""); @@ -15,7 +15,7 @@ export function FileManager() { return (
-

File manager

+

Database

diff --git a/web/src/fileManager/DateList.tsx b/web/src/database/DateList.tsx similarity index 100% rename from web/src/fileManager/DateList.tsx rename to web/src/database/DateList.tsx diff --git a/web/src/fileManager/FetchFiles.tsx b/web/src/database/FetchFiles.tsx similarity index 100% rename from web/src/fileManager/FetchFiles.tsx rename to web/src/database/FetchFiles.tsx diff --git a/web/src/fileManager/FileContent.tsx b/web/src/database/FileContent.tsx similarity index 100% rename from web/src/fileManager/FileContent.tsx rename to web/src/database/FileContent.tsx diff --git a/web/src/fileManager/FileNameList.tsx b/web/src/database/FileNameList.tsx similarity index 100% rename from web/src/fileManager/FileNameList.tsx rename to web/src/database/FileNameList.tsx diff --git a/web/src/fileManager/PrettifyCSV.tsx b/web/src/database/PrettifyCSV.tsx similarity index 100% rename from web/src/fileManager/PrettifyCSV.tsx rename to web/src/database/PrettifyCSV.tsx