mykola/mptv
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

make one big upsert to update channels using batchMerge. very fast compared to doing every insert individually

Browse source
This commit is contained in:
mykola2312 2024-04-22 03:47:05 +03:00
parent 62a1013cd7
commit 116f276c21
2 changed files with 40 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
src/main/java/com/mykola2312/mptv/db/M3ULoader.java
View file

@ -3,12 +3,14 @@ package com.mykola2312.mptv.db;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import org.jooq.UpdatableRecord;
import org.jooq.exception.NoDataFoundException; import org.jooq.exception.NoDataFoundException;
import org.jooq.impl.*; import org.jooq.impl.*;
import static com.mykola2312.mptv.tables.Category.*; import static com.mykola2312.mptv.tables.Category.*;
import static com.mykola2312.mptv.tables.Channel.*; import static com.mykola2312.mptv.tables.Channel.*;
import com.mykola2312.mptv.parser.M3U; import com.mykola2312.mptv.parser.M3U;
import com.mykola2312.mptv.tables.records.ChannelRecord;
public class M3ULoader { public class M3ULoader {
private static Integer ensureRootCategory(String rootName) { private static Integer ensureRootCategory(String rootName) {
@ -30,62 +32,51 @@ public class M3ULoader {
public static void loadAll(ArrayList<M3U> items, String rootName) { public static void loadAll(ArrayList<M3U> items, String rootName) {
Integer rootCategoryId = ensureRootCategory(rootName); Integer rootCategoryId = ensureRootCategory(rootName);
// cache categories' ids
HashMap<String, Integer> categories = new HashMap<>(); HashMap<String, Integer> categories = new HashMap<>();
// collect all groups, find or create them, cache their ids
for (M3U item : items) { for (M3U item : items) {
// category // no category, skip
Integer categoryId; if (item.groupTitle == null) {
if (item.groupTitle != null) { continue;
categoryId = categories.get(item.groupTitle); }
if (categoryId == null) { // we already have category cached
Integer id; if (categories.get(item.groupTitle) != null) {
try { continue;
id = DSL.using(DB.CONFIG)
.select(CATEGORY.ID)
.from(CATEGORY)
.where(CATEGORY.TITLE.eq(item.groupTitle))
.limit(1)
.fetchSingleInto(Integer.class);
} catch (NoDataFoundException e) {
id = DSL.using(DB.CONFIG)
.insertInto(CATEGORY, CATEGORY.TITLE)
.values(item.groupTitle)
.returningResult(CATEGORY.ID)
.fetchOne()
.into(Integer.class);
}
categories.put(item.groupTitle, id);
categoryId = id;
}
} else {
categoryId = rootCategoryId;
} }
// channel Integer categoryId;
try { try {
Integer channelId = DSL.using(DB.CONFIG) categoryId = DSL.using(DB.CONFIG)
.select(CHANNEL.ID) .select(CATEGORY.ID)
.from(CHANNEL) .from(CATEGORY)
.where(CHANNEL.CATEGORY.eq(categoryId) .where(CATEGORY.TITLE.eq(item.groupTitle))
.and(CHANNEL.TITLE.eq(item.title)))
.limit(1) .limit(1)
.fetchSingleInto(Integer.class); .fetchSingleInto(Integer.class);
DSL.using(DB.CONFIG)
.update(CHANNEL)
.set(CHANNEL.URL, item.url)
.set(CHANNEL.LOGO, item.tvgLogo)
.where(CHANNEL.ID.eq(channelId))
.execute();
} catch (NoDataFoundException e) { } catch (NoDataFoundException e) {
DSL.using(DB.CONFIG) categoryId = DSL.using(DB.CONFIG)
.insertInto(CHANNEL, .insertInto(CATEGORY, CATEGORY.TITLE)
CHANNEL.CATEGORY, CHANNEL.TITLE, .values(item.groupTitle)
CHANNEL.URL, CHANNEL.LOGO) .returningResult(CATEGORY.ID)
.values(categoryId, item.title, item.url, item.tvgLogo) .fetchSingleInto(Integer.class);
.execute();
} }
categories.put(item.groupTitle, categoryId);
} }
// upsert all channels
ArrayList<UpdatableRecord<ChannelRecord>> channels = new ArrayList<>();
for (M3U item : items) {
UpdatableRecord<ChannelRecord> channel = new UpdatableRecordImpl<>(CHANNEL);
Integer categoryId = item.groupTitle != null
? categories.get(item.groupTitle) : rootCategoryId;
channel.set(CHANNEL.CATEGORY, categoryId);
channel.set(CHANNEL.TITLE, item.title);
channel.set(CHANNEL.URL, item.url);
channel.set(CHANNEL.LOGO, item.tvgLogo);
channels.add(channel);
}
DSL.using(DB.CONFIG)
.batchMerge(channels)
.execute();
} }
} }

4
src/main/resources/db/migration/V001.01__init.sql
View file

@ -3,7 +3,7 @@ CREATE TABLE category (
title TEXT NOT NULL title TEXT NOT NULL
); );
CREATE INDEX idx_category_title ON category(title); CREATE UNIQUE INDEX idx_category_title ON category(title);
CREATE TABLE channel ( CREATE TABLE channel (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
@ -15,4 +15,4 @@ CREATE TABLE channel (
FOREIGN KEY (category) REFERENCES category(id) FOREIGN KEY (category) REFERENCES category(id)
); );
CREATE INDEX idx_channel_category_title ON channel(category,title); CREATE UNIQUE INDEX idx_channel_category_title ON channel(category,title);