diff --git a/pom.xml b/pom.xml
index d3400c8..b56131e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -144,6 +144,17 @@
org.jooq.meta.sqlite.SQLiteDatabase
.*
flyway_schema_history
+
+
+
+ CRAWLED_AT
+ BIGINT
+
+
+ LAST_TIME
+ BIGINT
+
+
true
diff --git a/src/main/java/com/mykola2312/mptv/Main.java b/src/main/java/com/mykola2312/mptv/Main.java
index 229b2cd..4c9fd6b 100644
--- a/src/main/java/com/mykola2312/mptv/Main.java
+++ b/src/main/java/com/mykola2312/mptv/Main.java
@@ -69,7 +69,8 @@ public class Main {
flyway.migrate();
// load sources, start crawlers
- Crawler crawler = new Crawler(config.sources);
+ Crawler crawler = new Crawler();
+ crawler.updateSources(config.sources);
crawler.crawl();
// initialize ui
diff --git a/src/main/java/com/mykola2312/mptv/crawler/Crawler.java b/src/main/java/com/mykola2312/mptv/crawler/Crawler.java
index fe3a76c..6452b9e 100644
--- a/src/main/java/com/mykola2312/mptv/crawler/Crawler.java
+++ b/src/main/java/com/mykola2312/mptv/crawler/Crawler.java
@@ -7,6 +7,7 @@ import org.apache.log4j.Logger;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.time.Instant;
import java.util.ArrayList;
import java.io.IOException;
import java.util.HashMap;
@@ -17,22 +18,23 @@ import org.jooq.impl.*;
import static com.mykola2312.mptv.tables.Category.*;
import static com.mykola2312.mptv.tables.Channel.*;
import static com.mykola2312.mptv.tables.Source.*;
+import static com.mykola2312.mptv.tables.Crawl.*;
import com.mykola2312.mptv.config.SourceItem;
import com.mykola2312.mptv.parser.M3U;
import com.mykola2312.mptv.parser.M3UException;
import com.mykola2312.mptv.parser.M3UParser;
import com.mykola2312.mptv.db.DB;
+import com.mykola2312.mptv.db.pojo.Source;
import com.mykola2312.mptv.tables.records.ChannelRecord;
import com.mykola2312.mptv.tables.records.SourceRecord;
public class Crawler {
private static final Logger logger = Logger.getLogger(Crawler.class);
- private final List sources;
+ private Integer crawlId;
- public Crawler(List sources) {
- this.sources = sources;
+ public Crawler() {
}
public void updateSources(List sourceItems) {
@@ -53,6 +55,13 @@ public class Crawler {
.execute();
}
+ private List loadSources() {
+ return DSL.using(DB.CONFIG)
+ .select()
+ .from(SOURCE)
+ .fetchInto(Source.class);
+ }
+
private Integer ensureRootCategory(String rootName) {
try {
return DSL.using(DB.CONFIG)
@@ -112,6 +121,7 @@ public class Crawler {
channel.set(CHANNEL.TITLE, item.title);
channel.set(CHANNEL.URL, item.url);
channel.set(CHANNEL.LOGO, item.tvgLogo);
+ channel.set(CHANNEL.CRAWL, crawlId);
channels.add(channel);
}
@@ -120,15 +130,24 @@ public class Crawler {
.execute();
}
+ private Integer beginCrawl() {
+ return DSL.using(DB.CONFIG)
+ .insertInto(CRAWL, CRAWL.CRAWLED_AT)
+ .values(Instant.now().toEpochMilli() / 1000L)
+ .returningResult(CRAWL.ID)
+ .fetchSingleInto(Integer.class);
+ }
+
public void crawl() {
- for (SourceItem source : sources) {
+ crawlId = beginCrawl();
+ for (Source source : loadSources()) {
switch (source.type) {
- case M3U_LOCAL -> {
+ case "m3u-local" -> {
try {
if (source.path == null) {
logger.error("m3u local has to have \"path\" variable");
continue;
- } else if (source.rootCategory == null) {
+ } else if (source.rootName == null) {
logger.error("source has to have \"rootCategory\"");
continue;
}
@@ -136,7 +155,7 @@ public class Crawler {
String m3uData = Files.readString(Paths.get(source.path), StandardCharsets.UTF_8);
ArrayList m3u = M3UParser.parse(m3uData);
- updateAllChannels(m3u, source.rootCategory);
+ updateAllChannels(m3u, source.rootName);
} catch (IOException e) {
logger.error(e);
logger.error(String.format("failed to read local m3u file: %s", e.getMessage()));
@@ -147,7 +166,7 @@ public class Crawler {
}
default -> {
- logger.error(String.format("source type %s is not implemented yet :(", source.type.name()));
+ logger.error(String.format("source type %s is not implemented yet :(", source.type));
}
}
}
diff --git a/src/main/java/com/mykola2312/mptv/db/pojo/Source.java b/src/main/java/com/mykola2312/mptv/db/pojo/Source.java
new file mode 100644
index 0000000..48191e0
--- /dev/null
+++ b/src/main/java/com/mykola2312/mptv/db/pojo/Source.java
@@ -0,0 +1,23 @@
+package com.mykola2312.mptv.db.pojo;
+
+import jakarta.persistence.Column;
+
+public class Source {
+ @Column(name = "ID")
+ public int id;
+
+ @Column(name = "TYPE")
+ public String type;
+
+ @Column(name = "ROOT_NAME")
+ public String rootName;
+
+ @Column(name = "URL")
+ public String url;
+
+ @Column(name = "PATH")
+ public String path;
+
+ @Column(name = "COOKIES")
+ public String cookies;
+}