/*
----------------------------------------------------------------------------------------------------
Module: TickDataSaverCustomBinary
Version: 3.18 (Simplified yearly-to-monthly migration: delete yearly on success, no marker file)
Author: cTrader Coding Assistant
Date: May 27, 2025

Description:
This cTrader cBot captures tick data. Data is cached and written to monthly files.
Output directory is a FULL PATH. If directory doesn't exist, bot stops.

One-Time Yearly to Monthly File Migration (OnStart, for years 2017-2025):
- Converts existing "Symbol_YYYY.tab" or "Symbol_YYYY.tab_zip" files into
  new uncompressed "Symbol_YYYY_MM.tab" files. Successfully migrated yearly files are DELETED.
  If monthly files for a year already exist, any corresponding yearly file is deleted.
  WARNING: This conversion can be memory-intensive for very large yearly files.

Post-Migration Operating Modes & Extensions: (Unchanged)
- Live Trading Output: ".tab-live" files (monthly, uncompressed).
- Historical Master Files:
    - ".tab" for the current system month (uncompressed, actively updated).
    - ".tab_zip" for months older than current system month (archived, compressed).
- Backtesting Output:
    - ".tab_zip" for months older than current system month (if zip doesn't already exist).
    - ".tab" (uncompressed) for the current system month.
...
----------------------------------------------------------------------------------------------------
*/
using System;
using System.IO;
using System.IO.Compression;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using cAlgo.API;
using cAlgo.API.Internals;

namespace cAlgo.Robots
{
    [Robot(TimeZone = TimeZones.UTC, AccessRights = AccessRights.FullAccess)]
    public class TickDataSaverCustomBinary : Robot
    {
        [Parameter("Output path", DefaultValue = "", Group = "Output Settings")]
        public string DirectoryName { get; set; }

        [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 16)]
        private struct TickDataRecord
        {
            public double OADateTime;
            public float Ask;
            public float Bid;
            public TickDataRecord(DateTime dateTime, float ask, float bid)
            { OADateTime = dateTime.ToOADate(); Ask = ask; Bid = bid; }
            public DateTime RecordTime => DateTime.FromOADate(OADateTime);
        }

        private List<TickDataRecord> _tickCache;
        private int _cachedDataYear = 0;
        private int _cachedDataMonth = 0;
        private string _dataFolderPath;
        private int _systemYearAtStart;
        private int _systemMonthAtStart;
        private string _outputCurrentModeFileExtension;
        private HashSet<string> _sessionOutputMonthsInitialized;
        private bool _isDirectoryValid = false;

        private DateTime GetLastTickTimeFromFile(string filePath) { /* ... (as in v3.17, unchanged) ... */
            if (!_isDirectoryValid) return DateTime.MinValue; 
            try
            {
                if (!File.Exists(filePath)) return DateTime.MinValue;
                using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                {
                    if (fs.Length < Marshal.SizeOf(typeof(TickDataRecord))) return DateTime.MinValue;
                    fs.Seek(-Marshal.SizeOf(typeof(TickDataRecord)), SeekOrigin.End); 
                    using (BinaryReader reader = new BinaryReader(fs))
                    {
                        double oaDate = reader.ReadDouble();
                        return DateTime.FromOADate(oaDate); 
                    }
                }
            }
            catch (Exception ex) { Print($"Error reading last tick time from {filePath}: {ex.Message}"); return DateTime.MinValue; }
        }
        
        private void CompressFileAndRemoveOriginal(string sourceTabFilePath, string targetZipFilePath, string entryNameInZip) { /* ... (as in v3.17, unchanged) ... */
            if (!_isDirectoryValid || !File.Exists(sourceTabFilePath))
            { Print($"Compression skipped: Source file '{sourceTabFilePath}' not found or directory invalid."); return; }
            Print($"Attempting to compress '{sourceTabFilePath}' to '{targetZipFilePath}'.");
            try
            {
                if (File.Exists(targetZipFilePath)) { File.Delete(targetZipFilePath); }
                using (FileStream originalFileStream = new FileStream(sourceTabFilePath, FileMode.Open, FileAccess.Read, FileShare.None))
                using (FileStream compressedFileStream = new FileStream(targetZipFilePath, FileMode.CreateNew))
                using (ZipArchive archive = new ZipArchive(compressedFileStream, ZipArchiveMode.Create))
                {
                    ZipArchiveEntry entry = archive.CreateEntry(entryNameInZip);
                    using (Stream entryStream = entry.Open()) { originalFileStream.CopyTo(entryStream); }
                }
                Print($"Successfully compressed '{sourceTabFilePath}' to '{targetZipFilePath}'.");
                try { File.Delete(sourceTabFilePath); Print($"Deleted original file '{sourceTabFilePath}'."); }
                catch (Exception exDel) { Print($"Error deleting original file '{sourceTabFilePath}' after compression: {exDel.Message}"); }
            }
            catch (IOException ioEx) when (File.Exists(targetZipFilePath) && ioEx.Message.Contains("already exists"))
            { Print($"Compressed file '{targetZipFilePath}' conflict during CreateNew. Error: {ioEx.Message}"); }
            catch (Exception ex)
            {
                Print($"Error during compression of '{sourceTabFilePath}': {ex.Message}");
                if (File.Exists(targetZipFilePath) ) { try { using(var fs = File.OpenRead(targetZipFilePath)) { if (fs.Length == 0) File.Delete(targetZipFilePath); } Print("Cleaned up potentially incomplete zip file on error."); } catch {} }
            }
        }

        private void ArchivePreExistingOldTabFile(int yearToProcess, int monthToProcess) { /* ... (as in v3.17, unchanged, operates on monthly .tab) ... */
            if (!_isDirectoryValid) return;
            string tabFileNameOnly = $"{Symbol.Name}_{yearToProcess}_{monthToProcess:D2}.tab"; 
            string tabFilePath = Path.Combine(_dataFolderPath, tabFileNameOnly);
            string zipFilePath = Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToProcess}_{monthToProcess:D2}.tab_zip"); 

            if (File.Exists(tabFilePath))
            {
                if (!File.Exists(zipFilePath)) 
                {
                    Print($"Found pre-existing .tab file for old month {yearToProcess}-{monthToProcess:D2}: '{tabFilePath}'. Archiving...");
                    CompressFileAndRemoveOriginal(tabFilePath, zipFilePath, tabFileNameOnly);
                } else {
                    Print($"Archive file '{zipFilePath}' already exists. Deleting redundant pre-existing .tab file '{tabFilePath}'.");
                    try { File.Delete(tabFilePath); } catch (Exception exDel) { Print($"Error deleting redundant .tab file '{tabFilePath}': {exDel.Message}"); }
                }
            }
        }
        
        private List<TickDataRecord> ReadAllRecordsFromStream(Stream stream) { /* ... (as in v3.17, unchanged) ... */
            var records = new List<TickDataRecord>();
            using (var reader = new BinaryReader(stream)) 
            {
                while (reader.BaseStream.Position < reader.BaseStream.Length) {
                    try {
                        records.Add(new TickDataRecord(DateTime.FromOADate(reader.ReadDouble()), reader.ReadSingle(), reader.ReadSingle()));
                    } catch (EndOfStreamException) { break; }
                      catch (Exception exRead) { Print($"Error reading record stream: {exRead.Message}"); break;}
                }
            }
            return records;
        }

        // Temporary one-time migration from Symbol_YYYY.ext to Symbol_YYYY_MM.tab
        private void MigrateYearlyFilesToMonthlyIfNeeded()
        {
            if (!_isDirectoryValid) return;
            Print("Starting one-time check to migrate old yearly files (2017-2025) to monthly .tab format...");

            for (int yearToMigrate = 2017; yearToMigrate <= 2025; yearToMigrate++)
            {
                string yearlyTabFileName = $"{Symbol.Name}_{yearToMigrate}.tab"; // Entry name if from zip
                string yearlyTabFilePath = Path.Combine(_dataFolderPath, yearlyTabFileName);
                string yearlyZipFilePath = Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToMigrate}.tab_zip");
                
                // Check if monthly files for this year ALREADY exist.
                bool monthlyFilesAlreadyExistForThisYear = false;
                for (int m = 1; m <= 12; m++)
                {
                    if (File.Exists(Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToMigrate}_{m:D2}.tab")) ||
                        File.Exists(Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToMigrate}_{m:D2}.tab_zip")))
                    {
                        monthlyFilesAlreadyExistForThisYear = true;
                        break;
                    }
                }

                if (monthlyFilesAlreadyExistForThisYear)
                {
                    Print($"Monthly files for year {yearToMigrate} already exist.");
                    // If monthly files exist, delete any remaining original yearly file for cleanup
                    if (File.Exists(yearlyTabFilePath))
                    {
                        Print($"Deleting obsolete yearly file: {yearlyTabFilePath}");
                        try { File.Delete(yearlyTabFilePath); } catch (Exception ex) { Print($"Error deleting {yearlyTabFilePath}: {ex.Message}"); }
                    }
                    if (File.Exists(yearlyZipFilePath))
                    {
                        Print($"Deleting obsolete yearly archive: {yearlyZipFilePath}");
                        try { File.Delete(yearlyZipFilePath); } catch (Exception ex) { Print($"Error deleting {yearlyZipFilePath}: {ex.Message}"); }
                    }
                    continue; // Skip to next year
                }

                // If we reach here, no monthly files for yearToMigrate exist yet.
                // Now, try to find and process the yearly source file.
                List<TickDataRecord> recordsForYear = null;
                string sourceYearlyFileProcessed = null; // Path of the file that was read

                if (File.Exists(yearlyZipFilePath))
                {
                    Print($"Found yearly archive {yearlyZipFilePath} for migration. Decompressing...");
                    sourceYearlyFileProcessed = yearlyZipFilePath;
                    try {
                        using (FileStream zipToOpen = new FileStream(yearlyZipFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
                        using (ZipArchive archive = new ZipArchive(zipToOpen, ZipArchiveMode.Read)) {
                            // Assume the entry name inside the zip is the original .tab filename
                            ZipArchiveEntry entry = archive.Entries.FirstOrDefault(e => e.Name.Equals(yearlyTabFileName, StringComparison.OrdinalIgnoreCase));
                            if (entry != null) {
                                using (Stream entryStream = entry.Open()) { recordsForYear = ReadAllRecordsFromStream(entryStream); }
                            } else { Print($"Entry '{yearlyTabFileName}' not found in {yearlyZipFilePath}."); }
                        }
                    } catch (Exception ex) { Print($"Error decompressing or reading {yearlyZipFilePath}: {ex.Message}"); }
                }
                else if (File.Exists(yearlyTabFilePath))
                {
                    Print($"Found yearly .tab file {yearlyTabFilePath} for migration. Reading...");
                    sourceYearlyFileProcessed = yearlyTabFilePath;
                    try {
                         using (FileStream fs = File.OpenRead(yearlyTabFilePath)) { recordsForYear = ReadAllRecordsFromStream(fs); }
                    } catch (Exception ex) { Print($"Error reading {yearlyTabFilePath}: {ex.Message}"); }
                }

                if (recordsForYear == null || !recordsForYear.Any())
                {
                    if (sourceYearlyFileProcessed != null) Print($"No records in {sourceYearlyFileProcessed} or read error for year {yearToMigrate}. Source file will not be deleted automatically if it was not processed.");
                    else Print($"No yearly file (.tab or .tab_zip) found for {yearToMigrate} to convert.");
                    continue; // Move to next year
                }

                Print($"Distributing {recordsForYear.Count} records from year {yearToMigrate} (source: {Path.GetFileName(sourceYearlyFileProcessed)}) into monthly .tab files. WARNING: This may be memory intensive.");
                var monthlyGroups = recordsForYear.GroupBy(r => r.RecordTime.Month);
                bool conversionSuccessAllMonths = true;

                foreach (var group in monthlyGroups)
                {
                    int month = group.Key;
                    string monthlyTabTargetName = $"{Symbol.Name}_{yearToMigrate}_{month:D2}.tab";
                    string monthlyTabTargetPath = Path.Combine(_dataFolderPath, monthlyTabTargetName);
                    List<TickDataRecord> monthRecords = group.OrderBy(r => r.RecordTime).ToList();

                    Print($"Writing {monthRecords.Count} records for {yearToMigrate}-{month:D2} to {monthlyTabTargetPath} (Mode: Create).");
                    if (!WriteRawCacheDataToFile(monthlyTabTargetPath, FileMode.Create, monthRecords)) 
                    {
                        conversionSuccessAllMonths = false;
                        Print($"Failed to write {monthlyTabTargetPath}. Yearly migration for {yearToMigrate} will be considered incomplete.");
                        break; 
                    }
                }

                if (conversionSuccessAllMonths)
                {
                    Print($"Successfully converted all months for year {yearToMigrate} from {sourceYearlyFileProcessed}. Deleting original yearly file.");
                    try { File.Delete(sourceYearlyFileProcessed); } 
                    catch (Exception ex) { Print($"Error deleting original yearly file {sourceYearlyFileProcessed} after successful conversion: {ex.Message}."); }
                }
                else
                {
                    Print($"Conversion of {sourceYearlyFileProcessed} for year {yearToMigrate} was incomplete due to errors. Original yearly file NOT deleted.");
                }
            }
            Print("One-time yearly to monthly file migration check finished.");
        }
        
        private void UpdateHistoricalTabFile(int yearToUpdate, int monthToUpdate) { /* ... (as in v3.17, unchanged) ... */
            if (!_isDirectoryValid) { Print($"Skipping UpdateHistoricalTabFile for {yearToUpdate}-{monthToUpdate:D2} as output directory is invalid."); return; }
            Print($"Attempting to update historical .tab file for month {yearToUpdate}-{monthToUpdate:D2} in: '{_dataFolderPath}'");
            string historyTabFilePath = Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToUpdate}_{monthToUpdate:D2}.tab"); 
            string zipFilePath = Path.ChangeExtension(historyTabFilePath, ".tab_zip");
            if (File.Exists(zipFilePath)) { Print($"Skipping update for {historyTabFilePath} as a compressed version '{zipFilePath}' already exists (month likely archived)."); return; }
            DateTime lastTickTimeInTabFile = GetLastTickTimeFromFile(historyTabFilePath);
            if (lastTickTimeInTabFile != DateTime.MinValue) Print($"Last tick in {historyTabFilePath} is at: {lastTickTimeInTabFile:yyyy-MM-dd HH:mm:ss.fff} UTC");
            else Print($"No existing .tab file found for {yearToUpdate}-{monthToUpdate:D2} or it's empty: {historyTabFilePath}");
            Ticks serverTicks = MarketData.GetTicks(SymbolName);
            List<TickDataRecord> recordsToAppendToTab = new List<TickDataRecord>();
            DateTime earliestTickAvailableInCollection = DateTime.MaxValue;
            if (serverTicks.Any()) { earliestTickAvailableInCollection = serverTicks.OrderBy(t => t.Time).First().Time; Print($"Earliest tick from server: {earliestTickAvailableInCollection:yyyy-MM-dd HH:mm:ss.fff} UTC"); }
            if (lastTickTimeInTabFile != DateTime.MinValue && earliestTickAvailableInCollection > lastTickTimeInTabFile) {
                 Print($"Attempting to load more history for .tab. Last in file: {lastTickTimeInTabFile:yyyy-MM-dd HH:mm:ss.fff} UTC, earliest from server: {earliestTickAvailableInCollection:yyyy-MM-dd HH:mm:ss.fff} UTC");
                 const int maxLoadAttempts = 100;
                 int totalLoadedHistorically = 0;
                 for (int i = 0; i < maxLoadAttempts; i++) {
                     int newlyLoadedCount = 0;
                     try { newlyLoadedCount = serverTicks.LoadMoreHistory(); } catch(Exception e) { Print($"Error during LoadMoreHistory: {e.Message}"); break; }
                     if (newlyLoadedCount == 0 && i > 0) { Print("LoadMoreHistory returned 0 new ticks in this attempt."); break; } 
                     totalLoadedHistorically += newlyLoadedCount;
                     if (!serverTicks.Any()) { Print("Ticks collection became empty after LoadMoreHistory."); break; }
                     earliestTickAvailableInCollection = serverTicks.OrderBy(t => t.Time).First().Time;
                     Print($"Total {totalLoadedHistorically} ticks potentially loaded via LoadMoreHistory. Earliest tick now: {earliestTickAvailableInCollection:yyyy-MM-dd HH:mm:ss.fff} UTC");
                     if (earliestTickAvailableInCollection <= lastTickTimeInTabFile) { Print("Gap to .tab file successfully bridged or covered by loaded history."); break; }
                     if (i == maxLoadAttempts - 1) Print("Max history load attempts reached.");
                 }
                 if (earliestTickAvailableInCollection > lastTickTimeInTabFile) { 
                    string msg = $"Gap to .tab file for {yearToUpdate}-{monthToUpdate:D2} might still exist. Last in .tab: {lastTickTimeInTabFile:yyyy-MM-dd HH:mm:ss.fff} UTC, Earliest from server: {earliestTickAvailableInCollection:yyyy-MM-dd HH:mm:ss.fff} UTC";
                    Print($"WARNING: {msg}"); MessageBox.Show($"{msg}. Run backtest to rebuild history database.", $"Warning for {Symbol.Name}", MessageBoxButton.OK, MessageBoxImage.Warning, MessageBoxResult.OK);
                 } else if (earliestTickAvailableInCollection != DateTime.MaxValue) { Print($"History load seems to cover up to or before last tick in .tab file for {yearToUpdate}-{monthToUpdate:D2}."); }
            }
            var newHistoricalTicks = serverTicks.Where(t => t.Time.Year == yearToUpdate && t.Time.Month == monthToUpdate && t.Time > lastTickTimeInTabFile).OrderBy(t => t.Time);
            foreach (Tick tick in newHistoricalTicks) recordsToAppendToTab.Add(new TickDataRecord(tick.Time, (float)tick.Ask, (float)tick.Bid));
            if (recordsToAppendToTab.Any()) {
                Print($"Found {recordsToAppendToTab.Count} new historical records to append to {historyTabFilePath}.");
                try {
                    using (FileStream fs = new FileStream(historyTabFilePath, FileMode.Append, FileAccess.Write, FileShare.Read))
                    using (BinaryWriter writer = new BinaryWriter(fs)) { foreach (var record in recordsToAppendToTab) { writer.Write(record.OADateTime); writer.Write(record.Ask); writer.Write(record.Bid); } writer.Flush(); fs.Flush(true); }
                    Print($"Successfully appended {recordsToAppendToTab.Count} records to {historyTabFilePath}.");
                } catch (Exception ex) { Print($"Error appending to historical .tab file {historyTabFilePath}: {ex.Message}"); }
            } else { Print($"No new historical records found to append to .tab file for {yearToUpdate}-{monthToUpdate:D2}."); }
        }
        
        // Helper method to write a list of TickDataRecord to a specific file with a specific mode
        private bool WriteRawCacheDataToFile(string filePath, FileMode mode, List<TickDataRecord> cacheToFlush) { /* ... (as in v3.17, unchanged) ... */
            if (!_isDirectoryValid) { Print($"WriteRawCacheDataToFile: Skipped writing to '{filePath}' as output directory is invalid."); return false; }
            if (cacheToFlush == null || !cacheToFlush.Any()) { Print($"WriteRawCacheDataToFile: Cache provided for '{filePath}' is empty or null. No data written."); return true; }
            Print($"Attempting to write {cacheToFlush.Count} records to '{filePath}' using {mode}.");
            try {
                string fileDirectory = Path.GetDirectoryName(filePath);
                if (!string.IsNullOrEmpty(fileDirectory) && !Directory.Exists(fileDirectory)) { Print($"Attempting to create directory for file: {fileDirectory}"); Directory.CreateDirectory(fileDirectory); }
                using (FileStream fileStream = new FileStream(filePath, mode, FileAccess.Write, FileShare.Read))
                using (BinaryWriter writer = new BinaryWriter(fileStream)) { foreach (var recordInCache in cacheToFlush) { writer.Write(recordInCache.OADateTime); writer.Write(recordInCache.Ask); writer.Write(recordInCache.Bid); } writer.Flush(); fileStream.Flush(true); }
                Print($"{cacheToFlush.Count} records successfully written to '{filePath}'.");
                return true;
            } catch (DirectoryNotFoundException dnfe) { Print($"DIRECTORY NOT FOUND while writing to '{filePath}': {dnfe.Message}. Marking directory as invalid."); _isDirectoryValid = false; return false; }
              catch (IOException ex) { Print($"IO Error writing cache to '{filePath}' using {mode}: {ex.Message}"); return false; }
              catch (Exception ex) { Print($"Unexpected error writing cache to '{filePath}' using {mode}: {ex.Message}"); return false; }
        }

        protected override void OnStart() { /* ... (as in v3.17, with MigrateYearlyFilesToMonthlyIfNeeded call first) ... */
            _tickCache = new List<TickDataRecord>();
            _sessionOutputMonthsInitialized = new HashSet<string>(); 
            _dataFolderPath = DirectoryName; 
            
            if (string.IsNullOrWhiteSpace(_dataFolderPath)) { Print("FATAL ERROR: The 'Output path' parameter is not set."); _isDirectoryValid = false; Stop(); return; }
            if (!Directory.Exists(_dataFolderPath)) { Print($"FATAL ERROR: The specified Output path does not exist: '{_dataFolderPath}'. Stopping cBot."); _isDirectoryValid = false; Stop(); return; }
            else { _isDirectoryValid = true; Print($"Using output directory: '{_dataFolderPath}'"); }
            
            _systemYearAtStart = DateTime.Now.Year; 
            _systemMonthAtStart = DateTime.Now.Month;
            _outputCurrentModeFileExtension = IsBacktesting ? ".tab" : ".tab-live"; 

            Print("Custom binary tick data saver with caching started.");
            Print($"Actual system date at start (for context): {_systemYearAtStart}-{_systemMonthAtStart:D2}");
            Print("Operating Mode: {0}. File extension for output files this session: '{1}'", IsBacktesting ? "Backtesting" : "Live", _outputCurrentModeFileExtension);
            
            if (_isDirectoryValid) 
            {
                MigrateYearlyFilesToMonthlyIfNeeded(); // Perform one-time migration first

                if (!IsBacktesting) 
                {
                    Print("Live mode: Checking for pre-existing old monthly .tab files to archive...");
                    for (int year = 2017; year <= _systemYearAtStart; year++) {
                        for (int month = 1; month <= 12; month++) {
                            if (year < _systemYearAtStart || (year == _systemYearAtStart && month < _systemMonthAtStart)) {
                                ArchivePreExistingOldTabFile(year, month); 
                            }
                        }
                    }
                    Print($"Live mode: Initial update of historical .tab file for month {_systemYearAtStart}-{_systemMonthAtStart:D2}...");
                    UpdateHistoricalTabFile(_systemYearAtStart, _systemMonthAtStart); 
                    Print("Live mode: Re-initializing cache for .tab-live session file."); 
                    _tickCache.Clear(); _cachedDataYear = 0; _cachedDataMonth = 0; _sessionOutputMonthsInitialized.Clear(); 
                    Print("In live mode, current month's .tab-live data will be flushed on {0} bar close.", TimeFrame); 
                    Print("Hourly timer starting for background updates of current month's .tab historical file.");
                    Timer.Start(3600); 
                }
            } else { Print("File operations (conversion, live startup) skipped due to invalid output directory."); }
        }
        
        protected override void OnTimer() { /* ... (as in v3.17, unchanged) ... */
            if (!_isDirectoryValid || IsBacktesting) return; 
            int currentYear = DateTime.Now.Year; 
            int currentMonth = DateTime.Now.Month;
            Print($"Hourly timer triggered at {Server.Time:yyyy-MM-dd HH:mm:ss} UTC. Attempting to update historical .tab file for month {currentYear}-{currentMonth:D2}...");
            UpdateHistoricalTabFile(currentYear, currentMonth);
        }

        protected override void OnBar() { /* ... (as in v3.17, unchanged) ... */
            if (!_isDirectoryValid || IsBacktesting) return; 
            int currentProcessingYear = Server.Time.Year; 
            int currentProcessingMonth = Server.Time.Month;
            if (_cachedDataYear == currentProcessingYear && _cachedDataMonth == currentProcessingMonth && _tickCache.Any()) {
                Print($"Bar closed on {TimeFrame}. Flushing cache to '{_outputCurrentModeFileExtension}' for live month {_cachedDataYear}-{_cachedDataMonth:D2} ({_tickCache.Count} records) at {Server.Time:yyyy-MM-dd HH:mm:ss.fff} UTC.");
                if (FlushOutputCacheToFile(_cachedDataYear, _cachedDataMonth)) { _tickCache.Clear(); Print("Cache successfully flushed and cleared on bar close."); }
                else { Print($"Failed to write cache to '{_outputCurrentModeFileExtension}' for month {_cachedDataYear}-{_cachedDataMonth:D2} on bar close. Cache ({_tickCache.Count} records) retained.");}
            }
        }

        protected override void OnTick() { /* ... (as in v3.17, unchanged) ... */
            if (!_isDirectoryValid) return; 
            DateTime tickTime = Server.Time; 
            float askPrice = (float)Symbol.Ask;
            float bidPrice = (float)Symbol.Bid;
            int yearOfTick = tickTime.Year;
            int monthOfTick = tickTime.Month; 
            var record = new TickDataRecord(tickTime, askPrice, bidPrice); 
            bool monthOrYearChanged = (_cachedDataYear != 0 && (_cachedDataYear != yearOfTick || _cachedDataMonth != monthOfTick));
            if (monthOrYearChanged && _tickCache.Any()) {
                Print($"Date change for output cache (from {_cachedDataYear}-{_cachedDataMonth:D2} to {yearOfTick}-{monthOfTick:D2}). Flushing cache for {_cachedDataYear}-{_cachedDataMonth:D2} ({_tickCache.Count} records).");
                if (FlushOutputCacheToFile(_cachedDataYear, _cachedDataMonth)) _tickCache.Clear(); 
                else { Print($"CRITICAL: Failed to flush output cache for {_cachedDataYear}-{_cachedDataMonth:D2}. Data remains in cache. Tick for {yearOfTick}-{monthOfTick:D2} dropped."); return; }
            }
            _cachedDataYear = yearOfTick; 
            _cachedDataMonth = monthOfTick;
            _tickCache.Add(record);       
        }

        private bool FlushOutputCacheToFile(int yearToFlush, int monthToFlush) { /* ... (as in v3.17, unchanged) ... */
            if (!_isDirectoryValid) { Print($"Skipping FlushOutputCacheToFile for {yearToFlush}-{monthToFlush:D2}: directory invalid."); return false; }
            if (!_tickCache.Any()) { Print($"FlushOutputCacheToFile called for {yearToFlush}-{monthToFlush:D2}, but cache is empty."); return true; }
            string fileKey = $"{yearToFlush}{monthToFlush:D2}";
            bool success = false;
            if (IsBacktesting) {
                string uncompressedTabFileName = $"{Symbol.Name}_{yearToFlush}_{monthToFlush:D2}.tab";
                string uncompressedTabFilePath = Path.Combine(_dataFolderPath, uncompressedTabFileName);
                string compressedZipFileName = $"{Symbol.Name}_{yearToFlush}_{monthToFlush:D2}.tab_zip";
                string compressedZipFilePath = Path.Combine(_dataFolderPath, compressedZipFileName);
                bool isOlderThanCurrentSystemMonth = yearToFlush < _systemYearAtStart || (yearToFlush == _systemYearAtStart && monthToFlush < _systemMonthAtStart);
                if (isOlderThanCurrentSystemMonth) {
                    if (File.Exists(compressedZipFilePath)) { Print($"Backtesting: Compressed file '{compressedZipFilePath}' for older month {yearToFlush}-{monthToFlush:D2} already exists. No new file created."); success = true; }
                    else {
                        Print($"Backtesting: Generating data for older month {yearToFlush}-{monthToFlush:D2} to temporary file '{uncompressedTabFilePath}' for compression.");
                        if (WriteRawCacheDataToFile(uncompressedTabFilePath, FileMode.Create, _tickCache)) { CompressFileAndRemoveOriginal(uncompressedTabFilePath, compressedZipFilePath, uncompressedTabFileName); success = true; } 
                        else { Print($"Backtesting: Failed to write .tab for {yearToFlush}-{monthToFlush:D2}. Compression skipped."); }
                    }
                } else {
                    Print($"Backtesting: Generating uncompressed .tab file for current/future system month {yearToFlush}-{monthToFlush:D2}: '{uncompressedTabFilePath}'.");
                    success = WriteRawCacheDataToFile(uncompressedTabFilePath, FileMode.Create, _tickCache);
                }
            } else {
                string outputFileName = $"{Symbol.Name}_{yearToFlush}_{monthToFlush:D2}{_outputCurrentModeFileExtension}"; 
                string outputFilePath = Path.Combine(_dataFolderPath, outputFileName);
                FileMode mode = _sessionOutputMonthsInitialized.Contains(fileKey) ? FileMode.Append : FileMode.Create;
                Print($"Mode for output file {outputFilePath} (Live .tab-live): {mode}.");
                success = WriteRawCacheDataToFile(outputFilePath, mode, _tickCache);
                if (success && mode == FileMode.Create) { _sessionOutputMonthsInitialized.Add(fileKey); }
            }
            return success;
        }

        protected override void OnStop() { /* ... (as in v3.17, unchanged) ... */
            Print("Stopping cBot...");
            if (!IsBacktesting && _isDirectoryValid) { Timer.Stop(); Print("Hourly timer for .tab file updates stopped."); }
            if (_isDirectoryValid && _tickCache.Any()) { 
                Print($"OnStop: Flushing remaining {_tickCache.Count} records from output cache for month {_cachedDataYear}-{_cachedDataMonth:D2} (Target ext: '{_outputCurrentModeFileExtension}').");
                if (FlushOutputCacheToFile(_cachedDataYear, _cachedDataMonth)) { _tickCache.Clear(); Print("Output cache successfully flushed and cleared."); }
                else { Print($"OnStop: Failed to flush remaining output cache for month {_cachedDataYear}-{_cachedDataMonth:D2}. {_tickCache.Count} records may be lost."); }
            } else if (!_isDirectoryValid) { Print("OnStop: No data flushed as output directory was invalid."); }
            else { Print("OnStop: No data in output cache to flush."); }
            _sessionOutputMonthsInitialized.Clear(); 
            Print("TickDataSaverCustomBinary stopped.");
        }
    }
}