diff --git a/StreamMaster.Infrastructure.EF.PGSQL/Scripts/012_migrate_new_channel_ids.sql b/StreamMaster.Infrastructure.EF.PGSQL/Scripts/012_migrate_new_channel_ids.sql new file mode 100644 index 000000000..381dac53c --- /dev/null +++ b/StreamMaster.Infrastructure.EF.PGSQL/Scripts/012_migrate_new_channel_ids.sql @@ -0,0 +1,115 @@ +BEGIN; + +-- Check if migration has already been performed +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM "SystemKeyValues" WHERE "Key" = 'didIDMigration') THEN + RAISE NOTICE 'Migration already completed.'; + RETURN; + END IF; +END $$; + +-- Create temporary tables for streams and m3ufiles data +CREATE TEMP TABLE temp_SMStreams AS +SELECT "Id", "Url", "CUID", "ChannelId", "EPGID", "TVGName", "Name", "M3UFileId" +FROM "SMStreams"; + +CREATE TEMP TABLE temp_M3UFiles AS +SELECT "Id", COALESCE("M3UKey", 0) AS "M3UKey" +FROM "M3UFiles"; + +-- Function to generate MD5 hash +CREATE OR REPLACE FUNCTION generate_md5(key TEXT, M3UFileId INT) +RETURNS TEXT AS $$ +DECLARE + hash TEXT; +BEGIN + SELECT md5(concat(key, '_', M3UFileId)) INTO hash; + RETURN hash; +END; +$$ LANGUAGE plpgsql; + +-- Function to generate M3UKey value +CREATE OR REPLACE FUNCTION generate_m3u_key_value(M3UKey INT, M3UFileId INT, Url TEXT, + CUID TEXT, ChannelId TEXT, EPGID TEXT, + TVGName TEXT, Name TEXT) +RETURNS TEXT AS $$ +DECLARE + key TEXT; +BEGIN + CASE M3UKey + WHEN 0 THEN key := Url; + WHEN 1 THEN key := CUID; + WHEN 2 THEN key := ChannelId; + WHEN 3 THEN key := EPGID; + WHEN 4 THEN key := COALESCE(TVGName, Name); + WHEN 5 THEN + IF TVGName IS NOT NULL AND EPGID IS NOT NULL THEN + key := TVGName || '_' || EPGID; + END IF; + WHEN 6 THEN key := Name; + WHEN 7 THEN + IF Name IS NOT NULL AND EPGID IS NOT NULL THEN + key := Name || '_' || EPGID; + END IF; + ELSE + RAISE EXCEPTION 'Invalid M3UKey value: %', M3UKey; + END CASE; + + IF key IS NOT NULL THEN + RETURN generate_md5(key, M3UFileId); + ELSE + RETURN NULL; + END IF; +END; +$$ LANGUAGE plpgsql; + +-- Create a temporary table for batch processing +CREATE TEMP TABLE temp_batch_update (old_id TEXT, new_id TEXT, m3ufileid INT); + +-- Insert new IDs into the batch update table +INSERT INTO temp_batch_update (old_id, new_id, m3ufileid) +SELECT s."Id", generate_m3u_key_value(f."M3UKey", s."M3UFileId", s."Url", s."CUID", + s."ChannelId", s."EPGID", s."TVGName", s."Name"), s."M3UFileId" +FROM temp_SMStreams s +LEFT JOIN temp_M3UFiles f ON s."M3UFileId" = f."Id" +WHERE s."M3UFileId" IS NOT NULL AND s."M3UFileId" >= 0; + +-- Update SMStreams with new IDs +INSERT INTO "SMStreams" ("Id", "ClientUserAgent", "FilePosition", "IsHidden", + "IsUserCreated", "M3UFileId", "ChannelNumber", + "M3UFileName", "Group", "EPGID", "Logo", "Name", + "Url", "StationId", "IsSystem", "CUID", "SMStreamType", + "NeedsDelete", "ChannelName", "ChannelId", + "CommandProfileName", "TVGName", "ExtInf") +SELECT t.new_id, s."ClientUserAgent", s."FilePosition", s."IsHidden", + s."IsUserCreated", t.m3ufileid, s."ChannelNumber", s."M3UFileName", + s."Group", s."EPGID", s."Logo", s."Name", s."Url", s."StationId", + s."IsSystem", s."CUID", s."SMStreamType", s."NeedsDelete", s."ChannelName", + s."ChannelId", s."CommandProfileName", s."TVGName", s."ExtInf" +FROM temp_batch_update t +INNER JOIN "SMStreams" s ON t.old_id = s."Id"; + +-- Update SMChannelStreamLinks with new IDs +INSERT INTO "SMChannelStreamLinks" ("SMStreamId", "SMChannelId", "Rank") +SELECT t.new_id, l."SMChannelId", l."Rank" +FROM temp_batch_update t +INNER JOIN "SMChannelStreamLinks" l ON t.old_id = l."SMStreamId"; + +-- Delete old SMChannelStreamLinks +DELETE FROM "SMChannelStreamLinks" +WHERE "SMStreamId" IN (SELECT old_id FROM temp_batch_update); + +-- Delete old SMStreams +DELETE FROM "SMStreams" +WHERE "Id" IN (SELECT old_id FROM temp_batch_update); + +-- Drop temporary tables +DROP TABLE temp_batch_update; +DROP TABLE temp_SMStreams; +DROP TABLE temp_M3UFiles; + +-- Add the didIDMigration entry to SystemKeyValues +INSERT INTO "SystemKeyValues" ("Key", "Value") VALUES ('didIDMigration', 'true'); + +COMMIT; \ No newline at end of file diff --git a/did_migrate_streams.sh b/did_migrate_streams.sh new file mode 100644 index 000000000..056ce63eb --- /dev/null +++ b/did_migrate_streams.sh @@ -0,0 +1,335 @@ +#!/bin/bash + +# Check if $PGDATA directory exists, if not set a default value +if [ ! -d "/config/DB" ]; then + echo "/config/DB directory does not exist. Assuming test" + . "/var/lib/postgresql/data/env.sh" + PGDATA=/var/lib/postgresql/data +else + . /env.sh +fi + +# Variables +batchSize=5000 # Increased batch size +dbDir="$PGDATA" +tempFileStreams="$dbDir/streams.csv" +tempFileM3UFiles="$dbDir/m3ufiles.csv" +tempFileBatch="$dbDir/batch_update.csv" +errorFile="$dbDir/did_errors.log" +tempTable="temp_batch_update" +delimiter='^' # Custom delimiter + +# Clean up any existing files +rm -f "$errorFile" "$tempFileStreams" "$tempFileM3UFiles" "$tempFileBatch" + +# Ensure PostgreSQL connection details are set +if [[ -z "$POSTGRES_USER" || -z "$POSTGRES_PASSWORD" || -z "$POSTGRES_DB" || -z "$POSTGRES_HOST" ]]; then + echo "Missing required PostgreSQL environment variables." + exit 1 +fi + +# Database connection command +PG_CMD="psql -h $POSTGRES_HOST -U $POSTGRES_USER -d $POSTGRES_DB" + +# Query to check if the record exists +checkMigration=$($PG_CMD -t -c "SELECT COUNT(*) FROM \"SystemKeyValues\" WHERE \"Key\" = 'didIDMigration';") +checkMigration=$(echo "$checkMigration" | xargs) # Trim whitespace + +if [[ "$checkMigration" -eq 0 ]]; then + echo "didIDMigration does not exist. Proceeding with migration..." +else + exit 0 +fi + +# Function to generate MD5 hash +generate_md5() { + local key=$1 + local M3UFileId=$2 + echo -n "${key}_${M3UFileId}" | md5sum | awk '{print $1}' +} + +# Function to generate M3UKey value +generate_m3u_key_value() { + local M3UKey=$1 + local M3UFileId=$2 + local Url=$3 + local CUID=$4 + local ChannelId=$5 + local EPGID=$6 + local TVGName=$7 + local Name=$8 + + local key + + case $M3UKey in + 0) key=$Url ;; + 1) key=$CUID ;; + 2) key=$ChannelId ;; + 3) key=$EPGID ;; + 4) key=${TVGName:-$Name} ;; + 5) + if [[ -n $TVGName && -n $EPGID ]]; then + key="${TVGName}_${EPGID}" + fi + ;; + 6) key=$Name ;; + 7) + if [[ -n $Name && -n $EPGID ]]; then + key="${Name}_${EPGID}" + fi + ;; + *) + echo "Invalid M3UKey value: $M3UKey" >&2 + ;; + esac + + if [[ -n $key ]]; then + generate_md5 "$key" "$M3UFileId" + else + echo "" + fi +} + +# Step 1: Fetch SMStreams and M3UFiles from PostgreSQL +echo "Fetching SMStreams and M3UFiles from the database..." +$PG_CMD -c "\COPY (SELECT \"Id\", \"Url\", \"CUID\", \"ChannelId\", \"EPGID\", \"TVGName\", \"Name\", \"M3UFileId\" FROM \"SMStreams\") TO '$tempFileStreams' WITH CSV HEADER DELIMITER '$delimiter';" +$PG_CMD -c "\COPY (SELECT \"Id\", COALESCE(\"M3UKey\", 0) AS \"M3UKey\" FROM \"M3UFiles\") TO '$tempFileM3UFiles' WITH CSV HEADER DELIMITER '$delimiter';" + +if [[ $? -ne 0 ]]; then + echo "Failed to fetch data from the database." + exit 1 +fi + +if [[ ! -f "$tempFileStreams" || ! -s "$tempFileStreams" ]]; then + echo "Error: Stream data file $tempFileStreams not created or is empty." + exit 1 +fi +echo "Stream data file $tempFileStreams fetched successfully." + +# Step 2: Build M3UFile mapping +declare -A m3uKeyMapping +while IFS="$delimiter" read -r Id M3UKey; do + m3uKeyMapping["$Id"]=$M3UKey + +done < <(tail -n +2 "$tempFileM3UFiles") # Skip the header line + +# Step 3: Process streams and generate new IDs +processedCount=0 +totalCount=$(wc -l <"$tempFileStreams") +((totalCount--)) # Subtract the header line +>"$tempFileBatch" + +while IFS="$delimiter" read -r Id Url CUID ChannelId EPGID TVGName Name M3UFileId; do + # Skip the header line + [[ "$Id" == "Id" ]] && continue + + # Handle edge cases where M3UFileId is empty or invalid + if [[ -z "$M3UFileId" || "$M3UFileId" -lt 0 ]]; then + M3UKey="0" + else + M3UKey=${m3uKeyMapping["$M3UFileId"]} + fi + + if [[ -z $M3UKey ]]; then + M3UKey="0" + fi + + # Generate the new ID + newId=$(generate_m3u_key_value "$M3UKey" "$M3UFileId" "$Url" "$CUID" "$ChannelId" "$EPGID" "$TVGName" "$Name") + + if [[ -n $newId ]]; then + echo "$Id,$newId,$M3UFileId" >>"$tempFileBatch" + ((processedCount++)) + fi + + # Process batch when size limit is reached + if [[ $((processedCount % batchSize)) -eq 0 ]]; then + echo "Updating batch of $batchSize records... (Processed: $processedCount/$totalCount)" + $PG_CMD <"$tempFileBatch" + fi +done < <(tail -n +2 "$tempFileStreams") # Skip the header line + +# Step 4: Update remaining records in the batch +if [[ -s "$tempFileBatch" ]]; then + echo "Updating final batch of records... (Processed: $processedCount/$totalCount)" + $PG_CMD <>"$errorFile" + exit 1 +fi + +echo "Migration completed successfully. Processed $processedCount streams." +echo "Temporary files retained in $dbDir."