mykola/mptv
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

move M3ULoader into crawler

Browse source
This commit is contained in:
mykola2312 2024-04-22 21:06:41 +03:00
parent 39a9020e31
commit 845d15fc3f
2 changed files with 77 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
src/main/java/com/mykola2312/mptv/crawler/Crawler.java
View file

@ -9,11 +9,20 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.io.IOException;
import java.util.HashMap;
import org.jooq.UpdatableRecord;
import org.jooq.exception.NoDataFoundException;
import org.jooq.impl.*;
import static com.mykola2312.mptv.tables.Category.*;
import static com.mykola2312.mptv.tables.Channel.*;
import com.mykola2312.mptv.config.SourceItem;
import com.mykola2312.mptv.parser.M3U;
import com.mykola2312.mptv.parser.M3UException;
import com.mykola2312.mptv.parser.M3UParser;
import com.mykola2312.mptv.db.DB;
import com.mykola2312.mptv.tables.records.ChannelRecord;
public class Crawler {
private static final Logger logger = Logger.getLogger(Crawler.class);
@ -24,6 +33,73 @@ public class Crawler {
this.sources = sources;
}
private static Integer ensureRootCategory(String rootName) {
try {
return DSL.using(DB.CONFIG)
.select(CATEGORY.ID)
.from(CATEGORY)
.where(CATEGORY.TITLE.eq(rootName))
.limit(1)
.fetchSingleInto(Integer.class);
} catch (NoDataFoundException e) {
return DSL.using(DB.CONFIG)
.insertInto(CATEGORY, CATEGORY.TITLE)
.values(rootName)
.returningResult(CATEGORY.ID)
.fetchSingleInto(Integer.class);
}
}
public static void loadAll(ArrayList<M3U> items, String rootName) {
Integer rootCategoryId = ensureRootCategory(rootName);
HashMap<String, Integer> categories = new HashMap<>();
// collect all groups, find or create them, cache their ids
for (M3U item : items) {
// no category, skip
if (item.groupTitle == null) {
continue;
}
// we already have category cached
if (categories.get(item.groupTitle) != null) {
continue;
}
Integer categoryId;
try {
categoryId = DSL.using(DB.CONFIG)
.select(CATEGORY.ID)
.from(CATEGORY)
.where(CATEGORY.TITLE.eq(item.groupTitle))
.limit(1)
.fetchSingleInto(Integer.class);
} catch (NoDataFoundException e) {
categoryId = DSL.using(DB.CONFIG)
.insertInto(CATEGORY, CATEGORY.TITLE)
.values(item.groupTitle)
.returningResult(CATEGORY.ID)
.fetchSingleInto(Integer.class);
}
categories.put(item.groupTitle, categoryId);
}
// upsert all channels
ArrayList<UpdatableRecord<ChannelRecord>> channels = new ArrayList<>();
for (M3U item : items) {
UpdatableRecord<ChannelRecord> channel = new UpdatableRecordImpl<>(CHANNEL);
Integer categoryId = item.groupTitle != null
? categories.get(item.groupTitle) : rootCategoryId;
channel.set(CHANNEL.CATEGORY, categoryId);
channel.set(CHANNEL.TITLE, item.title);
channel.set(CHANNEL.URL, item.url);
channel.set(CHANNEL.LOGO, item.tvgLogo);
channels.add(channel);
}
DSL.using(DB.CONFIG)
.batchMerge(channels)
.execute();
}
public void crawl() {
for (SourceItem source : sources) {
switch (source.type) {
@ -40,7 +116,7 @@ public class Crawler {
String m3uData = Files.readString(Paths.get(source.path), StandardCharsets.UTF_8);
ArrayList<M3U> m3u = M3UParser.parse(m3uData);
M3ULoader.loadAll(m3u, source.rootCategory);
loadAll(m3u, source.rootCategory);
} catch (IOException e) {
logger.error(e);
logger.error(String.format("failed to read local m3u file: %s", e.getMessage()));

83
src/main/java/com/mykola2312/mptv/crawler/M3ULoader.java
View file

@ -1,83 +0,0 @@
package com.mykola2312.mptv.crawler;
import java.util.ArrayList;
import java.util.HashMap;
import org.jooq.UpdatableRecord;
import org.jooq.exception.NoDataFoundException;
import org.jooq.impl.*;
import static com.mykola2312.mptv.tables.Category.*;
import static com.mykola2312.mptv.tables.Channel.*;
import com.mykola2312.mptv.db.DB;
import com.mykola2312.mptv.parser.M3U;
import com.mykola2312.mptv.tables.records.ChannelRecord;
public class M3ULoader {
private static Integer ensureRootCategory(String rootName) {
try {
return DSL.using(DB.CONFIG)
.select(CATEGORY.ID)
.from(CATEGORY)
.where(CATEGORY.TITLE.eq(rootName))
.limit(1)
.fetchSingleInto(Integer.class);
} catch (NoDataFoundException e) {
return DSL.using(DB.CONFIG)
.insertInto(CATEGORY, CATEGORY.TITLE)
.values(rootName)
.returningResult(CATEGORY.ID)
.fetchSingleInto(Integer.class);
}
}
public static void loadAll(ArrayList<M3U> items, String rootName) {
Integer rootCategoryId = ensureRootCategory(rootName);
HashMap<String, Integer> categories = new HashMap<>();
// collect all groups, find or create them, cache their ids
for (M3U item : items) {
// no category, skip
if (item.groupTitle == null) {
continue;
}
// we already have category cached
if (categories.get(item.groupTitle) != null) {
continue;
}
Integer categoryId;
try {
categoryId = DSL.using(DB.CONFIG)
.select(CATEGORY.ID)
.from(CATEGORY)
.where(CATEGORY.TITLE.eq(item.groupTitle))
.limit(1)
.fetchSingleInto(Integer.class);
} catch (NoDataFoundException e) {
categoryId = DSL.using(DB.CONFIG)
.insertInto(CATEGORY, CATEGORY.TITLE)
.values(item.groupTitle)
.returningResult(CATEGORY.ID)
.fetchSingleInto(Integer.class);
}
categories.put(item.groupTitle, categoryId);
}
// upsert all channels
ArrayList<UpdatableRecord<ChannelRecord>> channels = new ArrayList<>();
for (M3U item : items) {
UpdatableRecord<ChannelRecord> channel = new UpdatableRecordImpl<>(CHANNEL);
Integer categoryId = item.groupTitle != null
? categories.get(item.groupTitle) : rootCategoryId;
channel.set(CHANNEL.CATEGORY, categoryId);
channel.set(CHANNEL.TITLE, item.title);
channel.set(CHANNEL.URL, item.url);
channel.set(CHANNEL.LOGO, item.tvgLogo);
channels.add(channel);
}
DSL.using(DB.CONFIG)
.batchMerge(channels)
.execute();
}
}