Fix hardcoded 2023 year
This commit is contained in:
@@ -15,6 +15,7 @@ import parse.Aggregate.{AggregateKey, AggregateValue, DoubleValue, TimeDoubleLis
|
|||||||
import parse.{Parser, WeatherStationData}
|
import parse.{Parser, WeatherStationData}
|
||||||
|
|
||||||
import java.time.temporal.ChronoUnit
|
import java.time.temporal.ChronoUnit
|
||||||
|
import scala.util.matching.Regex
|
||||||
|
|
||||||
object PostgresService {
|
object PostgresService {
|
||||||
def of(transactor: Transactor[IO]): IO[PostgresService] = {
|
def of(transactor: Transactor[IO]): IO[PostgresService] = {
|
||||||
@@ -24,13 +25,25 @@ object PostgresService {
|
|||||||
|
|
||||||
class PostgresService(transactor: Transactor[IO], log: Logger[IO]) {
|
class PostgresService(transactor: Transactor[IO], log: Logger[IO]) {
|
||||||
def save(fileName: String, content: String): IO[String] = {
|
def save(fileName: String, content: String): IO[String] = {
|
||||||
val strLines = content.split(System.lineSeparator()).toList
|
val YearPattern: Regex = """(\d{4})\d{4}_\d{4}\.csv""".r
|
||||||
val weatherStationData = strLines.flatMap(Parser.parseLine)
|
|
||||||
insertInWeatherTable(weatherStationData)
|
def extractYear(filename: String): Option[String] = {
|
||||||
.attempt
|
YearPattern.findFirstMatchIn(filename).map(_.group(1))
|
||||||
.flatMap {
|
}
|
||||||
case Left(error) => log.error(s"Write db '$fileName' failed with error: ${error.getMessage}") *> IO.raiseError(error)
|
|
||||||
case Right(rowCount) => log.info(s"write rows: $rowCount file: $fileName").as(fileName)
|
extractYear(fileName) match {
|
||||||
|
case Some(year) =>
|
||||||
|
val strLines = content.split(System.lineSeparator()).toList
|
||||||
|
val weatherStationData = strLines.flatMap(line => Parser.parseLine(line, year))
|
||||||
|
insertInWeatherTable(weatherStationData)
|
||||||
|
.attempt
|
||||||
|
.flatMap {
|
||||||
|
case Left(error) => log.error(s"Write db '$fileName' failed with error: ${error.getMessage}") *> IO.raiseError(error)
|
||||||
|
case Right(rowCount) => log.info(s"write rows: $rowCount file: $fileName in year: $year").as(fileName)
|
||||||
|
}
|
||||||
|
|
||||||
|
case None =>
|
||||||
|
IO.raiseError(new RuntimeException(s"Could not extract year from file name: $fileName"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,20 +1,16 @@
|
|||||||
package parse
|
package parse
|
||||||
|
|
||||||
import parse.Aggregate.{AggregateValue, UserQuery, aggregateDoubleValues, aggregatePhenomenaValues, aggregateByField}
|
|
||||||
|
|
||||||
import java.time.LocalDateTime
|
import java.time.LocalDateTime
|
||||||
import java.time.format.DateTimeFormatter
|
import java.time.format.DateTimeFormatter
|
||||||
import java.time.temporal.ChronoUnit
|
|
||||||
import scala.util.Try
|
import scala.util.Try
|
||||||
|
|
||||||
object Parser {
|
object Parser {
|
||||||
def parseLine(line: String): Option[WeatherStationData] = {
|
def parseLine(line: String, year: String): Option[WeatherStationData] = {
|
||||||
val paramCount = WeatherData.getDoubleParamCount
|
val paramCount = WeatherData.getDoubleParamCount
|
||||||
|
|
||||||
def parseTimestamp(timestampStr: String): Option[LocalDateTime] = {
|
def parseTimestamp(timestampStr: String): Option[LocalDateTime] = {
|
||||||
val formatter = DateTimeFormatter.ofPattern("yyyydd.MM HH:mm")
|
val formatter = DateTimeFormatter.ofPattern("yyyydd.MM HH:mm")
|
||||||
// TODO figure out what to do with hardcoded year. Proly fetched data should be also modified to include year
|
Try(LocalDateTime.parse(s"$year${timestampStr.trim}", formatter)).toEither match {
|
||||||
Try(LocalDateTime.parse(s"2023${timestampStr.trim}", formatter)).toEither match {
|
|
||||||
case Right(timestamp) => Some(timestamp)
|
case Right(timestamp) => Some(timestamp)
|
||||||
case Left(_) => None
|
case Left(_) => None
|
||||||
}
|
}
|
||||||
@@ -35,25 +31,4 @@ object Parser {
|
|||||||
weatherData <- WeatherData.fromDoubles(strList.map(_.toDoubleOption), phenomena)
|
weatherData <- WeatherData.fromDoubles(strList.map(_.toDoubleOption), phenomena)
|
||||||
} yield WeatherStationData(city, timestamp, weatherData)
|
} yield WeatherStationData(city, timestamp, weatherData)
|
||||||
}
|
}
|
||||||
|
|
||||||
def queryData(userQuery: UserQuery, lines: List[String]): Map[String, Option[AggregateValue]] = {
|
|
||||||
val weatherByCity = lines
|
|
||||||
.flatMap(parseLine)
|
|
||||||
.filter(line => userQuery.cities.toList.contains(line.city))
|
|
||||||
.groupBy(_.city)
|
|
||||||
|
|
||||||
weatherByCity.map { case (city, weatherStationData) =>
|
|
||||||
userQuery.field match {
|
|
||||||
case "phenomena" => {
|
|
||||||
val phenomenaList = weatherStationData.map(_.weather.phenomena)
|
|
||||||
(city -> aggregatePhenomenaValues(userQuery.key, phenomenaList))
|
|
||||||
}
|
|
||||||
case field => {
|
|
||||||
val doubleList = aggregateByField(field, userQuery.granularity, weatherStationData)
|
|
||||||
(city -> aggregateDoubleValues(userQuery.key, doubleList))
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user