diff --git a/src/main/java/com/mykola2312/mptv/crawler/Crawler.java b/src/main/java/com/mykola2312/mptv/crawler/Crawler.java index 6d8156a..052a1ef 100644 --- a/src/main/java/com/mykola2312/mptv/crawler/Crawler.java +++ b/src/main/java/com/mykola2312/mptv/crawler/Crawler.java @@ -9,11 +9,20 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.io.IOException; +import java.util.HashMap; + +import org.jooq.UpdatableRecord; +import org.jooq.exception.NoDataFoundException; +import org.jooq.impl.*; +import static com.mykola2312.mptv.tables.Category.*; +import static com.mykola2312.mptv.tables.Channel.*; import com.mykola2312.mptv.config.SourceItem; import com.mykola2312.mptv.parser.M3U; import com.mykola2312.mptv.parser.M3UException; import com.mykola2312.mptv.parser.M3UParser; +import com.mykola2312.mptv.db.DB; +import com.mykola2312.mptv.tables.records.ChannelRecord; public class Crawler { private static final Logger logger = Logger.getLogger(Crawler.class); @@ -24,6 +33,73 @@ public class Crawler { this.sources = sources; } + private static Integer ensureRootCategory(String rootName) { + try { + return DSL.using(DB.CONFIG) + .select(CATEGORY.ID) + .from(CATEGORY) + .where(CATEGORY.TITLE.eq(rootName)) + .limit(1) + .fetchSingleInto(Integer.class); + } catch (NoDataFoundException e) { + return DSL.using(DB.CONFIG) + .insertInto(CATEGORY, CATEGORY.TITLE) + .values(rootName) + .returningResult(CATEGORY.ID) + .fetchSingleInto(Integer.class); + } + } + + public static void loadAll(ArrayList items, String rootName) { + Integer rootCategoryId = ensureRootCategory(rootName); + HashMap categories = new HashMap<>(); + // collect all groups, find or create them, cache their ids + for (M3U item : items) { + // no category, skip + if (item.groupTitle == null) { + continue; + } + // we already have category cached + if (categories.get(item.groupTitle) != null) { + continue; + } + + Integer categoryId; + try { + categoryId = DSL.using(DB.CONFIG) + .select(CATEGORY.ID) + .from(CATEGORY) + .where(CATEGORY.TITLE.eq(item.groupTitle)) + .limit(1) + .fetchSingleInto(Integer.class); + } catch (NoDataFoundException e) { + categoryId = DSL.using(DB.CONFIG) + .insertInto(CATEGORY, CATEGORY.TITLE) + .values(item.groupTitle) + .returningResult(CATEGORY.ID) + .fetchSingleInto(Integer.class); + } + categories.put(item.groupTitle, categoryId); + } + + // upsert all channels + ArrayList> channels = new ArrayList<>(); + for (M3U item : items) { + UpdatableRecord channel = new UpdatableRecordImpl<>(CHANNEL); + Integer categoryId = item.groupTitle != null + ? categories.get(item.groupTitle) : rootCategoryId; + channel.set(CHANNEL.CATEGORY, categoryId); + channel.set(CHANNEL.TITLE, item.title); + channel.set(CHANNEL.URL, item.url); + channel.set(CHANNEL.LOGO, item.tvgLogo); + + channels.add(channel); + } + DSL.using(DB.CONFIG) + .batchMerge(channels) + .execute(); + } + public void crawl() { for (SourceItem source : sources) { switch (source.type) { @@ -40,7 +116,7 @@ public class Crawler { String m3uData = Files.readString(Paths.get(source.path), StandardCharsets.UTF_8); ArrayList m3u = M3UParser.parse(m3uData); - M3ULoader.loadAll(m3u, source.rootCategory); + loadAll(m3u, source.rootCategory); } catch (IOException e) { logger.error(e); logger.error(String.format("failed to read local m3u file: %s", e.getMessage())); diff --git a/src/main/java/com/mykola2312/mptv/crawler/M3ULoader.java b/src/main/java/com/mykola2312/mptv/crawler/M3ULoader.java deleted file mode 100644 index d6ffadc..0000000 --- a/src/main/java/com/mykola2312/mptv/crawler/M3ULoader.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.mykola2312.mptv.crawler; - -import java.util.ArrayList; -import java.util.HashMap; - -import org.jooq.UpdatableRecord; -import org.jooq.exception.NoDataFoundException; -import org.jooq.impl.*; -import static com.mykola2312.mptv.tables.Category.*; -import static com.mykola2312.mptv.tables.Channel.*; - -import com.mykola2312.mptv.db.DB; -import com.mykola2312.mptv.parser.M3U; -import com.mykola2312.mptv.tables.records.ChannelRecord; - -public class M3ULoader { - private static Integer ensureRootCategory(String rootName) { - try { - return DSL.using(DB.CONFIG) - .select(CATEGORY.ID) - .from(CATEGORY) - .where(CATEGORY.TITLE.eq(rootName)) - .limit(1) - .fetchSingleInto(Integer.class); - } catch (NoDataFoundException e) { - return DSL.using(DB.CONFIG) - .insertInto(CATEGORY, CATEGORY.TITLE) - .values(rootName) - .returningResult(CATEGORY.ID) - .fetchSingleInto(Integer.class); - } - } - - public static void loadAll(ArrayList items, String rootName) { - Integer rootCategoryId = ensureRootCategory(rootName); - HashMap categories = new HashMap<>(); - // collect all groups, find or create them, cache their ids - for (M3U item : items) { - // no category, skip - if (item.groupTitle == null) { - continue; - } - // we already have category cached - if (categories.get(item.groupTitle) != null) { - continue; - } - - Integer categoryId; - try { - categoryId = DSL.using(DB.CONFIG) - .select(CATEGORY.ID) - .from(CATEGORY) - .where(CATEGORY.TITLE.eq(item.groupTitle)) - .limit(1) - .fetchSingleInto(Integer.class); - } catch (NoDataFoundException e) { - categoryId = DSL.using(DB.CONFIG) - .insertInto(CATEGORY, CATEGORY.TITLE) - .values(item.groupTitle) - .returningResult(CATEGORY.ID) - .fetchSingleInto(Integer.class); - } - categories.put(item.groupTitle, categoryId); - } - - // upsert all channels - ArrayList> channels = new ArrayList<>(); - for (M3U item : items) { - UpdatableRecord channel = new UpdatableRecordImpl<>(CHANNEL); - Integer categoryId = item.groupTitle != null - ? categories.get(item.groupTitle) : rootCategoryId; - channel.set(CHANNEL.CATEGORY, categoryId); - channel.set(CHANNEL.TITLE, item.title); - channel.set(CHANNEL.URL, item.url); - channel.set(CHANNEL.LOGO, item.tvgLogo); - - channels.add(channel); - } - DSL.using(DB.CONFIG) - .batchMerge(channels) - .execute(); - } -}