mykola/mptv
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

make one big upsert to update channels using batchMerge. very fast compared to doing every insert individually

Browse source
This commit is contained in:
mykola2312 2024-04-22 03:47:05 +03:00
parent 62a1013cd7
commit 116f276c21
2 changed files with 40 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
src/main/java/com/mykola2312/mptv/db/M3ULoader.java
View file

@ -3,12 +3,14 @@ package com.mykola2312.mptv.db;
import java.util.ArrayList;
import java.util.HashMap;
import org.jooq.UpdatableRecord;
import org.jooq.exception.NoDataFoundException;
import org.jooq.impl.*;
import static com.mykola2312.mptv.tables.Category.*;
import static com.mykola2312.mptv.tables.Channel.*;
import com.mykola2312.mptv.parser.M3U;
import com.mykola2312.mptv.tables.records.ChannelRecord;
public class M3ULoader {
private static Integer ensureRootCategory(String rootName) {
@ -30,62 +32,51 @@ public class M3ULoader {
public static void loadAll(ArrayList<M3U> items, String rootName) {
Integer rootCategoryId = ensureRootCategory(rootName);
// cache categories' ids
HashMap<String, Integer> categories = new HashMap<>();
// collect all groups, find or create them, cache their ids
for (M3U item : items) {
// category
Integer categoryId;
if (item.groupTitle != null) {
categoryId = categories.get(item.groupTitle);
if (categoryId == null) {
Integer id;
try {
id = DSL.using(DB.CONFIG)
.select(CATEGORY.ID)
.from(CATEGORY)
.where(CATEGORY.TITLE.eq(item.groupTitle))
.limit(1)
.fetchSingleInto(Integer.class);
} catch (NoDataFoundException e) {
id = DSL.using(DB.CONFIG)
.insertInto(CATEGORY, CATEGORY.TITLE)
.values(item.groupTitle)
.returningResult(CATEGORY.ID)
.fetchOne()
.into(Integer.class);
}
categories.put(item.groupTitle, id);
categoryId = id;
}
} else {
categoryId = rootCategoryId;
// no category, skip
if (item.groupTitle == null) {
continue;
}
// we already have category cached
if (categories.get(item.groupTitle) != null) {
continue;
}
// channel
Integer categoryId;
try {
Integer channelId = DSL.using(DB.CONFIG)
.select(CHANNEL.ID)
.from(CHANNEL)
.where(CHANNEL.CATEGORY.eq(categoryId)
.and(CHANNEL.TITLE.eq(item.title)))
categoryId = DSL.using(DB.CONFIG)
.select(CATEGORY.ID)
.from(CATEGORY)
.where(CATEGORY.TITLE.eq(item.groupTitle))
.limit(1)
.fetchSingleInto(Integer.class);
DSL.using(DB.CONFIG)
.update(CHANNEL)
.set(CHANNEL.URL, item.url)
.set(CHANNEL.LOGO, item.tvgLogo)
.where(CHANNEL.ID.eq(channelId))
.execute();
} catch (NoDataFoundException e) {
DSL.using(DB.CONFIG)
.insertInto(CHANNEL,
CHANNEL.CATEGORY, CHANNEL.TITLE,
CHANNEL.URL, CHANNEL.LOGO)
.values(categoryId, item.title, item.url, item.tvgLogo)
.execute();
categoryId = DSL.using(DB.CONFIG)
.insertInto(CATEGORY, CATEGORY.TITLE)
.values(item.groupTitle)
.returningResult(CATEGORY.ID)
.fetchSingleInto(Integer.class);
}
categories.put(item.groupTitle, categoryId);
}
// upsert all channels
ArrayList<UpdatableRecord<ChannelRecord>> channels = new ArrayList<>();
for (M3U item : items) {
UpdatableRecord<ChannelRecord> channel = new UpdatableRecordImpl<>(CHANNEL);
Integer categoryId = item.groupTitle != null
? categories.get(item.groupTitle) : rootCategoryId;
channel.set(CHANNEL.CATEGORY, categoryId);
channel.set(CHANNEL.TITLE, item.title);
channel.set(CHANNEL.URL, item.url);
channel.set(CHANNEL.LOGO, item.tvgLogo);
channels.add(channel);
}
DSL.using(DB.CONFIG)
.batchMerge(channels)
.execute();
}
}

4
src/main/resources/db/migration/V001.01__init.sql
View file

@ -3,7 +3,7 @@ CREATE TABLE category (
title TEXT NOT NULL
);
CREATE INDEX idx_category_title ON category(title);
CREATE UNIQUE INDEX idx_category_title ON category(title);
CREATE TABLE channel (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@ -15,4 +15,4 @@ CREATE TABLE channel (
FOREIGN KEY (category) REFERENCES category(id)
);
CREATE INDEX idx_channel_category_title ON channel(category,title);
CREATE UNIQUE INDEX idx_channel_category_title ON channel(category,title);