From b2b4eeff05e719b2232cef85bc207a975d8fbaa5 Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:24:45 -0500 Subject: [PATCH 01/10] Updated DBSyncWatcher Changed delegate to pass string instead of object. Now instead of attempting to deserialize the data in DBSyncWatcher, deserialization happens in th Syncer class. --- FreeTubeSyncer/Library/DBSyncWatcher.cs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/FreeTubeSyncer/Library/DBSyncWatcher.cs b/FreeTubeSyncer/Library/DBSyncWatcher.cs index dc656f8..d7a4246 100644 --- a/FreeTubeSyncer/Library/DBSyncWatcher.cs +++ b/FreeTubeSyncer/Library/DBSyncWatcher.cs @@ -10,7 +10,7 @@ public class DBSyncWatcher private FileSystemWatcher _watcher; public Dictionary WatchFiles { get; set; } = []; - public delegate void DatabaseChange(string db, object value); + public delegate void DatabaseChange(string db, string value); public event DatabaseChange OnDatabaseChange; public DBSyncWatcher(string path) @@ -43,9 +43,7 @@ public class DBSyncWatcher { if (line == "") continue; var type = WatchFiles[dbName]; - var item = JsonSerializer.Deserialize(line, type); - if (item == null) continue; - OnDatabaseChange?.Invoke(dbName, item); + OnDatabaseChange?.Invoke(dbName, line); } } @@ -58,9 +56,7 @@ public class DBSyncWatcher foreach (var line in data.Split('\n')) { var type = WatchFiles[dbName]; - var item = JsonSerializer.Deserialize(line, type); - if (item == null) continue; - OnDatabaseChange?.Invoke(dbName, item); + OnDatabaseChange?.Invoke(dbName, line); } } From 102a1d5d621a728834cab7f97b42960583a03e22 Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:25:25 -0500 Subject: [PATCH 02/10] Updated IDataModel Added two functions, MarshalData() and JsonData() to handle deserialization and serialization. (Special case with Setting) --- FreeTubeSyncer/Models/DatabaseModels/IDataModel.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/FreeTubeSyncer/Models/DatabaseModels/IDataModel.cs b/FreeTubeSyncer/Models/DatabaseModels/IDataModel.cs index 32f5341..f6c7672 100644 --- a/FreeTubeSyncer/Models/DatabaseModels/IDataModel.cs +++ b/FreeTubeSyncer/Models/DatabaseModels/IDataModel.cs @@ -4,4 +4,6 @@ public interface IDataModel { string Id(); bool EqualId(string oid); + void MarshalData(string id, string data); + string JsonData(); } \ No newline at end of file From 679449e4ecba0414f66a1356e19c0474d6cc02eb Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:25:59 -0500 Subject: [PATCH 03/10] Updated History Updated History to implement MarshalData() and JsonData() functions. --- .../Models/DatabaseModels/History.cs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/FreeTubeSyncer/Models/DatabaseModels/History.cs b/FreeTubeSyncer/Models/DatabaseModels/History.cs index 17afb97..7630133 100644 --- a/FreeTubeSyncer/Models/DatabaseModels/History.cs +++ b/FreeTubeSyncer/Models/DatabaseModels/History.cs @@ -1,4 +1,6 @@ using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using FreeTubeSyncer.Library; namespace FreeTubeSyncer.Models.DatabaseModels; @@ -23,4 +25,31 @@ public class History : IDataModel public string Id() => _id; public bool EqualId(string oid) => _id == oid; + + public void MarshalData(string id, string data) + { + var tobject = JsonSerializer.Deserialize(data, GlobalJsonOptions.Options); + if (tobject == null) + return; + this._id = id; + this.videoId = tobject.videoId; + this.title = tobject.title; + this.author = tobject.author; + this.authorId = tobject.authorId; + this.published = tobject.published; + this.description = tobject.description; + this.viewCount = tobject.viewCount; + this.lengthSeconds = tobject.lengthSeconds; + this.watchProgress = tobject.watchProgress; + this.timeWatched = tobject.timeWatched; + this.isLive = tobject.isLive; + this.type = tobject.type; + this.lastViewedPlaylistType = tobject.lastViewedPlaylistType; + this.lastViewedPlaylistItemId = tobject.lastViewedPlaylistItemId; + } + + public string JsonData() + { + return JsonSerializer.Serialize(this); + } } \ No newline at end of file From cce80d1d373430748fa54396e5574d08a7000354 Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:26:28 -0500 Subject: [PATCH 04/10] Updated Playlist Updated to implement MarshalData() and JsonData() --- .../Models/DatabaseModels/Playlist.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/FreeTubeSyncer/Models/DatabaseModels/Playlist.cs b/FreeTubeSyncer/Models/DatabaseModels/Playlist.cs index e4b20ef..ff04601 100644 --- a/FreeTubeSyncer/Models/DatabaseModels/Playlist.cs +++ b/FreeTubeSyncer/Models/DatabaseModels/Playlist.cs @@ -1,5 +1,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using FreeTubeSyncer.Library; namespace FreeTubeSyncer.Models.DatabaseModels; @@ -15,4 +17,21 @@ public class Playlist : IDataModel public string Id() => _id; public bool EqualId(string oid) => _id == oid; + public void MarshalData(string id, string data) + { + var tobject = JsonSerializer.Deserialize(data, GlobalJsonOptions.Options); + if (tobject == null) + return; + this._id = id; + this.playlistName = tobject.playlistName; + this.@protected = tobject.@protected; + this.videos = tobject.videos; + this.createdAt = tobject.createdAt; + this.lastUpdatedAt = tobject.lastUpdatedAt; + } + + public string JsonData() + { + return JsonSerializer.Serialize(this); + } } \ No newline at end of file From ae9a1d14bf27e2ad32da17af49d5a49dc89cfce9 Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:26:46 -0500 Subject: [PATCH 05/10] Updated Profile Added functions for MarshalData() and JsonData() --- .../Models/DatabaseModels/Profile.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/FreeTubeSyncer/Models/DatabaseModels/Profile.cs b/FreeTubeSyncer/Models/DatabaseModels/Profile.cs index eb5bcc4..e965a04 100644 --- a/FreeTubeSyncer/Models/DatabaseModels/Profile.cs +++ b/FreeTubeSyncer/Models/DatabaseModels/Profile.cs @@ -1,5 +1,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using FreeTubeSyncer.Library; namespace FreeTubeSyncer.Models.DatabaseModels; @@ -14,4 +16,22 @@ public class Profile : IDataModel public string Id() => _id; public bool EqualId(string oid) => _id == oid; + public void MarshalData(string id, string data) + { + var tobject = JsonSerializer.Deserialize(data, GlobalJsonOptions.Options); + if (tobject == null) + return; + this._id = tobject._id; + this.name = tobject.name; + this.bgColor = tobject.bgColor; + this.textColor = tobject.textColor; + this.subscriptions = tobject.subscriptions; + } + + public string JsonData() + { + return JsonSerializer.Serialize(this); + } + + public string IdName() => "_id"; } \ No newline at end of file From 25676baca4542cb2187a6807f5c3002c316b7583 Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:27:56 -0500 Subject: [PATCH 06/10] Updated SearchHistory Added MarshalData() and JsonData() functions. --- .../Models/DatabaseModels/SearchHistory.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/FreeTubeSyncer/Models/DatabaseModels/SearchHistory.cs b/FreeTubeSyncer/Models/DatabaseModels/SearchHistory.cs index e8bce80..93ddada 100644 --- a/FreeTubeSyncer/Models/DatabaseModels/SearchHistory.cs +++ b/FreeTubeSyncer/Models/DatabaseModels/SearchHistory.cs @@ -1,4 +1,6 @@ using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using FreeTubeSyncer.Library; namespace FreeTubeSyncer.Models.DatabaseModels; @@ -10,4 +12,17 @@ public class SearchHistory : IDataModel public string Id() => _id; public bool EqualId(string oid) => _id == oid; + public void MarshalData(string id, string data) + { + var tobject = JsonSerializer.Deserialize(data, GlobalJsonOptions.Options); + if (tobject == null) + return; + this._id = tobject._id; + this.lastUpdatedAt = tobject.lastUpdatedAt; + } + + public string JsonData() + { + return JsonSerializer.Serialize(this); + } } \ No newline at end of file From 09cd72b278eb15ec42e97f1b11fe623b10b9fe5a Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:28:06 -0500 Subject: [PATCH 07/10] Updated Profile removed un-used function. --- FreeTubeSyncer/Models/DatabaseModels/Profile.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/FreeTubeSyncer/Models/DatabaseModels/Profile.cs b/FreeTubeSyncer/Models/DatabaseModels/Profile.cs index e965a04..c97530e 100644 --- a/FreeTubeSyncer/Models/DatabaseModels/Profile.cs +++ b/FreeTubeSyncer/Models/DatabaseModels/Profile.cs @@ -32,6 +32,4 @@ public class Profile : IDataModel { return JsonSerializer.Serialize(this); } - - public string IdName() => "_id"; } \ No newline at end of file From 8e65cc3a4b82e2047a1f2310383a00250ad4dfef Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:29:13 -0500 Subject: [PATCH 08/10] Updated Setting Updated Seting class to use string for teh data, and the data is the full setting line, to allow for the flexability for Any data value in the field. Implemented MarshalData() and JsonData() functions. --- .../Models/DatabaseModels/Setting.cs | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/FreeTubeSyncer/Models/DatabaseModels/Setting.cs b/FreeTubeSyncer/Models/DatabaseModels/Setting.cs index 68f7dfc..42af9ca 100644 --- a/FreeTubeSyncer/Models/DatabaseModels/Setting.cs +++ b/FreeTubeSyncer/Models/DatabaseModels/Setting.cs @@ -8,17 +8,19 @@ public class Setting : IDataModel { #pragma warning disable CS8618 public string _id { get; set; } = string.Empty; - public string? ValueJson { get; set; } -#pragma warning restore CS8618 - - public object Value - { -#pragma warning disable CS8603 - get => string.IsNullOrEmpty(ValueJson) ? null : JsonSerializer.Deserialize(ValueJson); -#pragma warning restore CS8603 - set => ValueJson = JsonSerializer.Serialize(value); - } - + public string value { get; set; } = string.Empty; + public string Id() => _id; public bool EqualId(string oid) => _id == oid; + + public void MarshalData(string id, string data) + { + _id = id; + value = data; + } + + public string JsonData() + { + return value; + } } \ No newline at end of file From fb8284b5e848d705b24a2e2a42d3d1c9d24728d2 Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:30:54 -0500 Subject: [PATCH 09/10] Updated Syncer Updated Syncer class to properly sync all classes, even with special requierments needed for Settings. Now properly handles all classes, and ensures that the data is properly stored in the database, and can be written out correctly to the FreeTube database files. --- FreeTubeSyncer/REST/Syncer.cs | 52 ++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/FreeTubeSyncer/REST/Syncer.cs b/FreeTubeSyncer/REST/Syncer.cs index b0fc24d..13758f9 100644 --- a/FreeTubeSyncer/REST/Syncer.cs +++ b/FreeTubeSyncer/REST/Syncer.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading.Tasks; using FreeTubeSyncer.Library; using FreeTubeSyncer.Models.DatabaseModels; @@ -10,11 +11,12 @@ using RestSharp; namespace FreeTubeSyncer.REST; -public class Syncer where T : class, IDataModel +public class Syncer where T : class, IDataModel, new() { private List _entries = new List(); private RestClient _client; private string _dbPath; + private string _dbName; private string _restEndpoint; public bool IsDirty = false; @@ -25,6 +27,7 @@ public class Syncer where T : class, IDataModel watcher.OnDatabaseChange += HandleDatabaseChange; _client = new RestClient(new RestClientOptions("http://localhost:5183")); _dbPath = dbPath; + _dbName = dbName; _restEndpoint = restEndpoint; ReadDatabase().Wait(); FetchDatabase().Wait(); @@ -36,20 +39,24 @@ public class Syncer where T : class, IDataModel foreach (var entry in lines) { if (entry == "") continue; + T? item; try { - var item = JsonSerializer.Deserialize(entry, GlobalJsonOptions.Options); - if (item == null) continue; - if (_entries.Any(x => x.EqualId(item.Id()))) - _entries.RemoveAll(x => x.EqualId(item.Id())); - _entries.Add(item); - Console.WriteLine($"Posting {item.Id()}"); - await _client.PostJsonAsync(_restEndpoint, item); + item = JsonSerializer.Deserialize(entry, GlobalJsonOptions.Options); } catch (Exception ex) { - Console.WriteLine($"Failed to parse: {entry}"); + var jobj = JsonSerializer.Deserialize(entry, GlobalJsonOptions.Options); + item = new T(); + item.MarshalData(jobj["_id"].GetValue(), entry); } + if (item == null) continue; + item.MarshalData(item.Id(), entry); + if (_entries.Any(x => x.EqualId(item.Id()))) + _entries.RemoveAll(x => x.EqualId(item.Id())); + _entries.Add(item); + Console.WriteLine($"Posting {item.Id()}"); + await _client.PostJsonAsync(_restEndpoint, item); } } @@ -66,9 +73,25 @@ public class Syncer where T : class, IDataModel } } - private async void HandleDatabaseChange(string dbName, object entryObject) + private async void HandleDatabaseChange(string dbName, string entryObject) { - var entry = (T)entryObject; + if (dbName != _dbName) + return; + + T? entry; + try + { + entry = JsonSerializer.Deserialize(entryObject, GlobalJsonOptions.Options); + } + catch (Exception ex) + { + var jobj = JsonSerializer.Deserialize(entryObject, GlobalJsonOptions.Options); + entry = new T(); + entry.MarshalData(jobj["_id"].GetValue(), entryObject); + } + + if (entry == null) return; + entry.MarshalData(entry.Id(), entryObject); if (_entries.Any(x => x.EqualId(entry.Id()))) _entries.RemoveAll(x => x.EqualId(entry.Id())); _entries.Add(entry); @@ -78,11 +101,14 @@ public class Syncer where T : class, IDataModel public void Sync() { + if (!IsDirty) + return; + Console.WriteLine($"Syncing {_dbPath}..."); var json = new List(); foreach (var entry in _entries) - json.Add(JsonSerializer.Serialize(entry)); + json.Add(entry.JsonData()); File.WriteAllLines(_dbPath, json); - Console.WriteLine($"Updated {_dbPath}"); + Console.WriteLine($"Updated {_dbPath}."); IsDirty = false; } } \ No newline at end of file From ea2fb4d8189adc82d7981b6d5cbe38da72222c5b Mon Sep 17 00:00:00 2001 From: Mario Steele Date: Thu, 24 Jul 2025 04:32:51 -0500 Subject: [PATCH 10/10] Updated Program.cs While still testing things out, remain in Console mode. Removed un-used using statements. Fixed SearchHistorySyner to properly use /searchHistory API end-point. Added print statement to see when FreeTube closes, and wait for 1.5 seconds before attempting to sync everything up. May need to increase this more, needs testing. (maybe a good 5 seconds would be enough) --- FreeTubeSyncer/Program.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/FreeTubeSyncer/Program.cs b/FreeTubeSyncer/Program.cs index 82b1176..24adb90 100644 --- a/FreeTubeSyncer/Program.cs +++ b/FreeTubeSyncer/Program.cs @@ -1,7 +1,5 @@ using Avalonia; using System; -using System.Collections; -using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Threading; @@ -40,7 +38,7 @@ class Program var historySyncer = new Syncer(dbWatcher, Path.Join(path, "history.db"), "history.db", "/history"); var playlistSyncer = new Syncer(dbWatcher, Path.Join(path, "playlists.db"), "playlists.db", "/playlist"); var profileSyncer = new Syncer(dbWatcher, Path.Join(path, "profiles.db"), "profiles.db", "/profile"); - var searchHistorySyncer = new Syncer(dbWatcher, Path.Join(path, "search-history.db"), "search-history.db", "/search"); + var searchHistorySyncer = new Syncer(dbWatcher, Path.Join(path, "search-history.db"), "search-history.db", "/searchHistory"); var settingsSyncer = new Syncer(dbWatcher, Path.Join(path, "settings.db"), "settings.db", "/settings"); Task.Run(() => CheckCanSync(historySyncer, playlistSyncer, profileSyncer, searchHistorySyncer, settingsSyncer)); @@ -56,6 +54,9 @@ class Program { Thread.Sleep(100); if (Process.GetProcessesByName("FreeTube").Length > 0) continue; + Console.WriteLine("FreeTube has closed, we're going to try and update."); + Thread.Sleep(1500); + if (historySyncer is { IsDirty: true }) historySyncer.Sync(); if (playlistSyncer is { IsDirty: true })