Initial commit with weather data fetcher and parser
This commit is contained in:
Guntis Smaukstelis
@@ -0,0 +1,12 @@
|
||||
<configuration>
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="STDOUT" />
|
||||
<!-- <logger name="org.http4s" level="INFO" /> -->
|
||||
</root>
|
||||
</configuration>
|
||||
@@ -0,0 +1,51 @@
|
||||
package fetch
|
||||
|
||||
import cats.effect._
|
||||
import cats.implicits._
|
||||
import org.http4s._
|
||||
import org.http4s.blaze.client.BlazeClientBuilder
|
||||
import org.http4s.client.Client
|
||||
import org.http4s.headers.Authorization
|
||||
|
||||
import com.typesafe.config.ConfigFactory
|
||||
|
||||
import java.nio.file.{Files, Paths}
|
||||
import scala.concurrent.ExecutionContext.global
|
||||
|
||||
object FetchData extends IOApp.Simple {
|
||||
private val config = ConfigFactory.load()
|
||||
private val basicCredentials = BasicCredentials(config.getString("username"), config.getString("password"))
|
||||
private val baseUrl = Uri.unsafeFromString(config.getString("url")) // 20220831_1330.csv
|
||||
// private val baseUrl = Uri.unsafeFromString("https://jsonplaceholder.typicode.com/") // todos/1
|
||||
|
||||
def saveToFile(fileName: String, content: String): IO[Unit] = {
|
||||
val path = Paths.get(s"data/$fileName")
|
||||
IO(Files.writeString(path, content)).attempt.flatMap {
|
||||
case Right(_) => IO(println(s"write: $fileName"))
|
||||
case Left(error) => IO(println(s"Write file '$fileName' failed with error: ${error.getMessage}"))
|
||||
}
|
||||
}
|
||||
|
||||
def makeRequest(client: Client[IO], url: Uri): IO[Unit] = {
|
||||
val fileName = url.path.toString()
|
||||
for {
|
||||
request <- Request[IO](Method.GET, url)
|
||||
.withHeaders(Authorization(basicCredentials))
|
||||
.pure[IO]
|
||||
responseOrError <- client.expect[String](request).attempt
|
||||
_ <- responseOrError match {
|
||||
case Right(response) => saveToFile(fileName, response)
|
||||
case Left(error) => IO(println(s"Request failed to url: $url with error: ${error.getMessage}"))
|
||||
}
|
||||
} yield ()
|
||||
}
|
||||
|
||||
def run: IO[Unit] = {
|
||||
val fileNames = FileName.generateLast5Hours()
|
||||
// fileNames.foreach(println)
|
||||
val urls = fileNames.map(baseUrl / _)
|
||||
BlazeClientBuilder[IO](global).resource.use { client =>
|
||||
urls.traverse(url => makeRequest(client, url)) // urls.map(...).sequence
|
||||
}
|
||||
}.as(ExitCode.Success)
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package fetch
|
||||
|
||||
import java.time.format.DateTimeFormatter
|
||||
import java.time.{LocalDateTime, Duration}
|
||||
|
||||
object FileName {
|
||||
private val interval = Duration.ofMinutes(30)
|
||||
private val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")s
|
||||
|
||||
def generateLastHour(): List[String] = {
|
||||
val now = LocalDateTime.now
|
||||
generate(now.minusHours(1), now)
|
||||
}
|
||||
|
||||
def generateLast5Hours(): List[String] = {
|
||||
val now = roundToInterval(LocalDateTime.now, false)
|
||||
generate(now.minusHours(5), now)
|
||||
}
|
||||
|
||||
private def roundToInterval(time: LocalDateTime, roundUp: Boolean): LocalDateTime = {
|
||||
val minutes = time.getMinute
|
||||
val adjustment = if (roundUp) (interval.toMinutes - minutes % interval.toMinutes) % interval.toMinutes
|
||||
else -minutes % interval.toMinutes
|
||||
|
||||
time.plusMinutes(adjustment)
|
||||
}
|
||||
|
||||
// not sure either data are each 10 minutes or each 30 mins of hour
|
||||
def generate(startTime: LocalDateTime, endTime: LocalDateTime): List[String] = {
|
||||
val roundStartTime = roundToInterval(startTime, true)
|
||||
val roundEndTime = roundToInterval(endTime, false)
|
||||
|
||||
Iterator.iterate(roundStartTime) { time =>
|
||||
time.plus(interval)
|
||||
}.takeWhile(!_.isAfter(roundEndTime))
|
||||
.filter(_.getMinute == 30)
|
||||
.map(time => s"${formatter.format(time)}.csv")
|
||||
.toList
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package parse
|
||||
|
||||
import java.time.LocalDateTime
|
||||
import scala.reflect.runtime.universe.{termNames, typeOf}
|
||||
|
||||
|
||||
case class WeatherStationData(
|
||||
city: String,
|
||||
timestamp: LocalDateTime,
|
||||
meteo: MeteoData,
|
||||
phenomena: List[String],
|
||||
)
|
||||
|
||||
case class MeteoData(
|
||||
tempMax: Option[Double],
|
||||
tempMin: Option[Double],
|
||||
tempAvg: Option[Double],
|
||||
precipitation: Option[Double],
|
||||
windSpeedAvg: Option[Double],
|
||||
windGustMax: Option[Double],
|
||||
visibilityMin: Option[Double],
|
||||
visibilityAvg: Option[Double],
|
||||
snowThicknessAvg: Option[Double],
|
||||
atmPressure: Option[Double],
|
||||
dewPoint: Option[Double],
|
||||
airHumidity: Option[Double],
|
||||
sunshineDuration: Option[Double]
|
||||
)
|
||||
|
||||
object MeteoData {
|
||||
def fromDoubles(data: List[Option[Double]]): Option[MeteoData] = data match {
|
||||
case List(
|
||||
tempMax,
|
||||
tempMin,
|
||||
tempAvg,
|
||||
precipitation,
|
||||
windSpeedAvg,
|
||||
windGustMax,
|
||||
visibilityMin,
|
||||
visibilityAvg,
|
||||
snowThicknessAvg,
|
||||
atmPressure,
|
||||
dewPoint,
|
||||
airHumidity,
|
||||
sunshineDuration
|
||||
) => Some(MeteoData(
|
||||
tempMax,
|
||||
tempMin,
|
||||
tempAvg,
|
||||
precipitation,
|
||||
windSpeedAvg,
|
||||
windGustMax,
|
||||
visibilityMin,
|
||||
visibilityAvg,
|
||||
snowThicknessAvg,
|
||||
atmPressure,
|
||||
dewPoint,
|
||||
airHumidity,
|
||||
sunshineDuration,
|
||||
))
|
||||
case _ => None
|
||||
}
|
||||
|
||||
def getCount: Int = {
|
||||
val constructor = typeOf[MeteoData].decl(termNames.CONSTRUCTOR).asMethod
|
||||
val paramCount = constructor.paramLists.flatten.size
|
||||
paramCount
|
||||
}
|
||||
|
||||
def getKeys: List[String] = {
|
||||
val constructor = typeOf[MeteoData].decl(termNames.CONSTRUCTOR).asMethod
|
||||
constructor.paramLists.flatten.map(_.name.toString)
|
||||
}
|
||||
}
|
||||
|
||||
//val cityList: List[String] = List("Ainaži", "Alūksne", "Bauska", "Dagda", "Daugavgrīva", "Daugavpils", "Dobele", "Gulbene", "Jelgava", "Kalnciems", "Kolka", "Kuldīga", "Lielpēči", "Liepāja", "Madona", "Mērsrags", "Pāvilosta", "Piedruja", "Priekuļi", "Rēzekne", "Rīga", "Rucava", "Rūjiena", "Saldus", "Sigulda", "Sīļi", "Skrīveri", "Skulte", "Stende", "Ventspils", "Vičaki", "Zīlāni", "Zosēni", "Ainaži", "Alūksne", "Bauska", "Dagda", "Daugavgrīva", "Daugavpils", "Dobele", "Gulbene", "Jelgava", "Kalnciems", "Kolka", "Kuldīga", "Lielpēči", "Liepāja", "Madona", "Mērsrags", "Pāvilosta", "Piedruja", "Priekuļi", "Rēzekne", "Rīga", "Rucava", "Rūjiena", "Saldus", "Sigulda", "Sīļi", "Skrīveri", "Skulte", "Stende", "Ventspils", "Vičaki", "Zīlāni", "Zosēni")
|
||||
|
||||
|
||||
//Map(
|
||||
// "tempMax" -> "Stundas maksimālā temperatūra",
|
||||
// "tempMin" -> "Stundas minimālā temperatūra",
|
||||
// "tempAvg" -> "Stundas vidējā temperatūra",
|
||||
// "precipitation" -> "Stundas nokrišņu daudzums",
|
||||
// "windSpeedAvg" -> "Vidējais vēja ātrums novērojumu termiņā (10 min. vidējais)",
|
||||
// "windGustMax" -> "Stundas maksimālās vēja brāzmas",
|
||||
// "visibilityMin" -> "Stundas minimālā redzamība",
|
||||
// "visibilityAvg" -> "Stundas vidējā redzamība",
|
||||
// "snowThicknessAvg" -> "Stundas vidējais sniega segas biezums",
|
||||
// "atmPressure" -> "Atmosfēras spiediens jūras līmenī novērojuma termiņā (milibāros)",
|
||||
// "dewPoint" -> "Rasas punkta temperatūra novērojuma termiņā",
|
||||
// "airHumidity" -> "Relatīvais gaisa mitrums novērojumu termiņā",
|
||||
// "sunshineDuration" -> "Saules spīdēšanas ilgums",
|
||||
//// "weatherPhenomena" -> "Laika parādības",
|
||||
//)
|
||||
@@ -0,0 +1,89 @@
|
||||
package parse
|
||||
|
||||
import java.io.File
|
||||
import java.time.LocalDateTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
import scala.io.Source
|
||||
import scala.util.Try
|
||||
|
||||
object Parser {
|
||||
val data_path = "/Users/guntissmaukstelis/sandbox/hello/data/"
|
||||
|
||||
def fileToDateTime(file: File): LocalDateTime = {
|
||||
val dateString = file.toString.split("/").last.split("\\.").head
|
||||
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
|
||||
LocalDateTime.parse(dateString, formatter)
|
||||
}
|
||||
|
||||
def getFiles(path: String): List[File] = new File(path).listFiles.toList
|
||||
|
||||
def getFilesInRange(start: LocalDateTime, end: LocalDateTime): List[File] = {
|
||||
val fileMap = getFiles(data_path).map(file => fileToDateTime(file) -> file).toMap
|
||||
fileMap
|
||||
.filter { case (k, _) =>
|
||||
k.plusSeconds(1).isAfter(start) && k.minusSeconds(1).isBefore(end)
|
||||
}
|
||||
.map { case (_, v) => v }.toList
|
||||
}
|
||||
|
||||
// TODO I guess this should be wrapped in IO
|
||||
def readFromFile(file: File): List[String] = {
|
||||
val source = Source.fromFile(file)
|
||||
val lineList = source.getLines.toList
|
||||
source.close()
|
||||
lineList.tail // remove header line
|
||||
}
|
||||
|
||||
def parseLine(line: String): Option[WeatherStationData] = {
|
||||
val paramCount = MeteoData.getCount
|
||||
|
||||
def parseTimestamp(timestampStr: String): Option[LocalDateTime] = {
|
||||
val formatter = DateTimeFormatter.ofPattern("yyyyMM.dd HH:mm")
|
||||
Try(LocalDateTime.parse(s"2023${timestampStr.trim}", formatter)).toEither match {
|
||||
case Right(timestamp) => Some(timestamp)
|
||||
case Left(_) => None
|
||||
}
|
||||
}
|
||||
|
||||
def splitParts(parts: List[String]): Option[(String, String, List[String])] = {
|
||||
parts.splitAt(2) match {
|
||||
case (cityStr :: timestampStr :: Nil, rest) if rest.size >= paramCount => Some((cityStr, timestampStr, rest))
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
(city, timestampStr, rest) <- splitParts(line.trim.split(";", -1).toList)
|
||||
timestamp <- parseTimestamp(timestampStr)
|
||||
(strList, meteoPhenomenaList) = rest.splitAt(paramCount)
|
||||
meteoData <- MeteoData.fromDoubles(strList.map(_.toDoubleOption))
|
||||
phenomena <- Some(meteoPhenomenaList.map(_.trim))
|
||||
} yield WeatherStationData(city, timestamp, meteoData, phenomena)
|
||||
}
|
||||
|
||||
def readFromVariable(str: String): List[String] = str.split("\n").toList // Data.csv
|
||||
|
||||
|
||||
def main(args: Array[String]): Unit = {
|
||||
println("================ start parser")
|
||||
|
||||
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
|
||||
val start = LocalDateTime.parse("20230409_2200", formatter)
|
||||
val end = LocalDateTime.parse("20230501_1230", formatter)
|
||||
val files = getFilesInRange(start, end)
|
||||
val weatherStationData = files
|
||||
.flatMap(readFromFile)
|
||||
.flatMap(parseLine)
|
||||
|
||||
// weatherStationData.foreach(println)
|
||||
|
||||
val liepajaWeather = weatherStationData.filter(_.city == "Liepāja")
|
||||
val maxTempLiepaja = liepajaWeather.flatMap(_.meteo.tempMax).max
|
||||
val minTempLiepaja = liepajaWeather.flatMap(_.meteo.tempMin).min
|
||||
val avgTemps = liepajaWeather.flatMap(_.meteo.tempAvg)
|
||||
val avgTempLiepaja = avgTemps.sum / avgTemps.size
|
||||
println(s"Liepaja max: $maxTempLiepaja, min: $minTempLiepaja, avg: $avgTempLiepaja")
|
||||
// liepaja.foreach(println)
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package server
|
||||
|
||||
class Server {
|
||||
// TODO proly BlazeServerBuilder which accepts requests from client and serves parsed data
|
||||
}
|
||||
Reference in New Issue
Block a user