|
|
@@ -12,7 +12,7 @@ import org.springframework.stereotype.Component
|
|
12
|
12
|
import java.io.File
|
|
13
|
13
|
import java.nio.charset.StandardCharsets
|
|
14
|
14
|
import java.nio.file.Paths
|
|
15
|
|
-import scala.collection.immutable
|
|
|
15
|
+import scala.collection.{immutable, mutable}
|
|
16
|
16
|
import scala.concurrent.Await
|
|
17
|
17
|
import scala.concurrent.duration.DurationInt
|
|
18
|
18
|
import scala.language.postfixOps
|
|
|
@@ -26,6 +26,7 @@ class MovieQueryService extends MovieServiceTrait {
|
|
26
|
26
|
val titleBasicsResource: File = new File("src/main/resources/title.basics.tsv.gz")
|
|
27
|
27
|
val titlePrincipalsResource: File = new File("src/main/resources/title.principals.tsv.gz")
|
|
28
|
28
|
val nameBasicsResource: File = new File("src/main/resources/name.basics.tsv.gz")
|
|
|
29
|
+ val titleEpisodeResource: File = new File("src/main/resources/title.episode.tsv.gz")
|
|
29
|
30
|
|
|
30
|
31
|
def fileSource(file: File): Source[ByteString, NotUsed] = {
|
|
31
|
32
|
Source.single(file)
|
|
|
@@ -40,6 +41,10 @@ class MovieQueryService extends MovieServiceTrait {
|
|
40
|
41
|
}
|
|
41
|
42
|
|
|
42
|
43
|
|
|
|
44
|
+ val res = getSeriesTitle(titleEpisodeResource)
|
|
|
45
|
+ val r = getTitleByIds(titleBasicsResource, res)
|
|
|
46
|
+ println(r)
|
|
|
47
|
+
|
|
43
|
48
|
override def principalsForMovieName(name: String): List[Principal] = {
|
|
44
|
49
|
|
|
45
|
50
|
val titleId: String = getIdOfTitle(titleBasicsResource, name)
|
|
|
@@ -49,6 +54,8 @@ class MovieQueryService extends MovieServiceTrait {
|
|
49
|
54
|
}
|
|
50
|
55
|
|
|
51
|
56
|
override def tvSeriesWithGreatestNumberOfEpisodes(): List[TvSeries] = ???
|
|
|
57
|
+ //val personsIdList: List[String] = getSeriestitle(titleEpisodeResource)
|
|
|
58
|
+
|
|
52
|
59
|
|
|
53
|
60
|
|
|
54
|
61
|
def getIdOfTitle(file: File, titleName: String): String = {
|
|
|
@@ -69,6 +76,23 @@ class MovieQueryService extends MovieServiceTrait {
|
|
69
|
76
|
result.value.get.get.get
|
|
70
|
77
|
}
|
|
71
|
78
|
|
|
|
79
|
+ def getTitleByIds(file: File, titleIdMap: mutable.Map[String, Int]): Seq[Option[String]] = {
|
|
|
80
|
+ val result = fileSource(file)
|
|
|
81
|
+ .mapAsync(50) {
|
|
|
82
|
+ res =>
|
|
|
83
|
+ Source.single(res)
|
|
|
84
|
+ .via(lineParser)
|
|
|
85
|
+ .filter(row => titleIdMap.contains(row.getOrElse("tconst", "")))
|
|
|
86
|
+ .map(a => a.get("primaryTitle"))
|
|
|
87
|
+ .withAttributes(supervisionStrategy(resumingDecider))
|
|
|
88
|
+ .runWith(Sink.collection)
|
|
|
89
|
+ }
|
|
|
90
|
+ .withAttributes(supervisionStrategy(resumingDecider))
|
|
|
91
|
+ .runWith(Sink.head)
|
|
|
92
|
+
|
|
|
93
|
+ Await.result(result, 5 minutes)
|
|
|
94
|
+ result.value.get.get.toList
|
|
|
95
|
+ }
|
|
72
|
96
|
def getIdOfPersons(file: File, titleId: String): immutable.Iterable[Option[String]] = {
|
|
73
|
97
|
val te = fileSource(file)
|
|
74
|
98
|
.mapAsync(50) {
|
|
|
@@ -101,4 +125,26 @@ class MovieQueryService extends MovieServiceTrait {
|
|
101
|
125
|
result.value.get.get.toList
|
|
102
|
126
|
}
|
|
103
|
127
|
|
|
|
128
|
+ def getSeriesTitle(file: File): mutable.Map[String, Int] = {
|
|
|
129
|
+
|
|
|
130
|
+ val seriesMap = collection.mutable.Map("0" -> 0)
|
|
|
131
|
+ val maxSeriesMap = collection.mutable.Map("0" -> 0)
|
|
|
132
|
+
|
|
|
133
|
+ val result = Source.single(file)
|
|
|
134
|
+ .flatMapConcat(f => FileIO.fromPath(Paths.get(f.getPath), 1 * 1024 * 1024))
|
|
|
135
|
+ .via(Compression.gunzip())
|
|
|
136
|
+ .via(CsvParsing.lineScanner(CsvParsing.Tab, CsvParsing.DoubleQuote, CsvParsing.DoubleQuote))
|
|
|
137
|
+ .via(CsvToMap.toMapAsStringsCombineAll(StandardCharsets.UTF_8, Option.empty))
|
|
|
138
|
+ .map(row => seriesMap += seriesMap.get(row("parentTconst")).map(x => row("parentTconst") -> (x + 1)).getOrElse(row("parentTconst") -> 1))
|
|
|
139
|
+ .runWith(Sink.seq)
|
|
|
140
|
+
|
|
|
141
|
+ Await.result(result, 5 minutes)
|
|
|
142
|
+
|
|
|
143
|
+ for (_ <- 1 to 10) {
|
|
|
144
|
+ maxSeriesMap.addOne(seriesMap.maxBy(x => x._2)._1 -> seriesMap.maxBy(x => x._2)._2)
|
|
|
145
|
+ seriesMap.remove(seriesMap.maxBy(x => x._2)._1)
|
|
|
146
|
+ }
|
|
|
147
|
+ maxSeriesMap
|
|
|
148
|
+ }
|
|
|
149
|
+
|
|
104
|
150
|
}
|