Initial commit

This commit is contained in:
Michael Schimmel
2026-01-28 09:10:52 +01:00
commit a1d2e96f8a
513 changed files with 19503 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
{"version":"2.0.0","tasks":[{"label":"build","command":"dotnet","type":"process","args":["build","${workspaceFolder}","/property:GenerateFullPaths=true","/consoleLoggerParameters:NoSummary"],"problemMatcher":"$msCompile"}]}
@@ -0,0 +1,22 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.30011.22
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MS_ExportTickData", "MS_ExportTickData\MS_ExportTickData.csproj", "{e689f11c-ce60-494c-a1f3-b269c1dfa127}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{e689f11c-ce60-494c-a1f3-b269c1dfa127}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{e689f11c-ce60-494c-a1f3-b269c1dfa127}.Debug|Any CPU.Build.0 = Debug|Any CPU
{e689f11c-ce60-494c-a1f3-b269c1dfa127}.Release|Any CPU.ActiveCfg = Release|Any CPU
{e689f11c-ce60-494c-a1f3-b269c1dfa127}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal
@@ -0,0 +1,446 @@
/*
----------------------------------------------------------------------------------------------------
Module: TickDataSaverCustomBinary
Version: 3.18 (Simplified yearly-to-monthly migration: delete yearly on success, no marker file)
Author: cTrader Coding Assistant
Date: May 27, 2025
Description:
This cTrader cBot captures tick data. Data is cached and written to monthly files.
Output directory is a FULL PATH. If directory doesn't exist, bot stops.
One-Time Yearly to Monthly File Migration (OnStart, for years 2017-2025):
- Converts existing "Symbol_YYYY.tab" or "Symbol_YYYY.tab_zip" files into
new uncompressed "Symbol_YYYY_MM.tab" files. Successfully migrated yearly files are DELETED.
If monthly files for a year already exist, any corresponding yearly file is deleted.
WARNING: This conversion can be memory-intensive for very large yearly files.
Post-Migration Operating Modes & Extensions: (Unchanged)
- Live Trading Output: ".tab-live" files (monthly, uncompressed).
- Historical Master Files:
- ".tab" for the current system month (uncompressed, actively updated).
- ".tab_zip" for months older than current system month (archived, compressed).
- Backtesting Output:
- ".tab_zip" for months older than current system month (if zip doesn't already exist).
- ".tab" (uncompressed) for the current system month.
...
----------------------------------------------------------------------------------------------------
*/
using System;
using System.IO;
using System.IO.Compression;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using cAlgo.API;
using cAlgo.API.Internals;
namespace cAlgo.Robots
{
[Robot(TimeZone = TimeZones.UTC, AccessRights = AccessRights.FullAccess)]
public class TickDataSaverCustomBinary : Robot
{
[Parameter("Output path", DefaultValue = "", Group = "Output Settings")]
public string DirectoryName { get; set; }
[StructLayout(LayoutKind.Sequential, Pack = 1, Size = 16)]
private struct TickDataRecord
{
public double OADateTime;
public float Ask;
public float Bid;
public TickDataRecord(DateTime dateTime, float ask, float bid)
{ OADateTime = dateTime.ToOADate(); Ask = ask; Bid = bid; }
public DateTime RecordTime => DateTime.FromOADate(OADateTime);
}
private List<TickDataRecord> _tickCache;
private int _cachedDataYear = 0;
private int _cachedDataMonth = 0;
private string _dataFolderPath;
private int _systemYearAtStart;
private int _systemMonthAtStart;
private string _outputCurrentModeFileExtension;
private HashSet<string> _sessionOutputMonthsInitialized;
private bool _isDirectoryValid = false;
private DateTime GetLastTickTimeFromFile(string filePath) { /* ... (as in v3.17, unchanged) ... */
if (!_isDirectoryValid) return DateTime.MinValue;
try
{
if (!File.Exists(filePath)) return DateTime.MinValue;
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
if (fs.Length < Marshal.SizeOf(typeof(TickDataRecord))) return DateTime.MinValue;
fs.Seek(-Marshal.SizeOf(typeof(TickDataRecord)), SeekOrigin.End);
using (BinaryReader reader = new BinaryReader(fs))
{
double oaDate = reader.ReadDouble();
return DateTime.FromOADate(oaDate);
}
}
}
catch (Exception ex) { Print($"Error reading last tick time from {filePath}: {ex.Message}"); return DateTime.MinValue; }
}
private void CompressFileAndRemoveOriginal(string sourceTabFilePath, string targetZipFilePath, string entryNameInZip) { /* ... (as in v3.17, unchanged) ... */
if (!_isDirectoryValid || !File.Exists(sourceTabFilePath))
{ Print($"Compression skipped: Source file '{sourceTabFilePath}' not found or directory invalid."); return; }
Print($"Attempting to compress '{sourceTabFilePath}' to '{targetZipFilePath}'.");
try
{
if (File.Exists(targetZipFilePath)) { File.Delete(targetZipFilePath); }
using (FileStream originalFileStream = new FileStream(sourceTabFilePath, FileMode.Open, FileAccess.Read, FileShare.None))
using (FileStream compressedFileStream = new FileStream(targetZipFilePath, FileMode.CreateNew))
using (ZipArchive archive = new ZipArchive(compressedFileStream, ZipArchiveMode.Create))
{
ZipArchiveEntry entry = archive.CreateEntry(entryNameInZip);
using (Stream entryStream = entry.Open()) { originalFileStream.CopyTo(entryStream); }
}
Print($"Successfully compressed '{sourceTabFilePath}' to '{targetZipFilePath}'.");
try { File.Delete(sourceTabFilePath); Print($"Deleted original file '{sourceTabFilePath}'."); }
catch (Exception exDel) { Print($"Error deleting original file '{sourceTabFilePath}' after compression: {exDel.Message}"); }
}
catch (IOException ioEx) when (File.Exists(targetZipFilePath) && ioEx.Message.Contains("already exists"))
{ Print($"Compressed file '{targetZipFilePath}' conflict during CreateNew. Error: {ioEx.Message}"); }
catch (Exception ex)
{
Print($"Error during compression of '{sourceTabFilePath}': {ex.Message}");
if (File.Exists(targetZipFilePath) ) { try { using(var fs = File.OpenRead(targetZipFilePath)) { if (fs.Length == 0) File.Delete(targetZipFilePath); } Print("Cleaned up potentially incomplete zip file on error."); } catch {} }
}
}
private void ArchivePreExistingOldTabFile(int yearToProcess, int monthToProcess) { /* ... (as in v3.17, unchanged, operates on monthly .tab) ... */
if (!_isDirectoryValid) return;
string tabFileNameOnly = $"{Symbol.Name}_{yearToProcess}_{monthToProcess:D2}.tab";
string tabFilePath = Path.Combine(_dataFolderPath, tabFileNameOnly);
string zipFilePath = Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToProcess}_{monthToProcess:D2}.tab_zip");
if (File.Exists(tabFilePath))
{
if (!File.Exists(zipFilePath))
{
Print($"Found pre-existing .tab file for old month {yearToProcess}-{monthToProcess:D2}: '{tabFilePath}'. Archiving...");
CompressFileAndRemoveOriginal(tabFilePath, zipFilePath, tabFileNameOnly);
} else {
Print($"Archive file '{zipFilePath}' already exists. Deleting redundant pre-existing .tab file '{tabFilePath}'.");
try { File.Delete(tabFilePath); } catch (Exception exDel) { Print($"Error deleting redundant .tab file '{tabFilePath}': {exDel.Message}"); }
}
}
}
private List<TickDataRecord> ReadAllRecordsFromStream(Stream stream) { /* ... (as in v3.17, unchanged) ... */
var records = new List<TickDataRecord>();
using (var reader = new BinaryReader(stream))
{
while (reader.BaseStream.Position < reader.BaseStream.Length) {
try {
records.Add(new TickDataRecord(DateTime.FromOADate(reader.ReadDouble()), reader.ReadSingle(), reader.ReadSingle()));
} catch (EndOfStreamException) { break; }
catch (Exception exRead) { Print($"Error reading record stream: {exRead.Message}"); break;}
}
}
return records;
}
// Temporary one-time migration from Symbol_YYYY.ext to Symbol_YYYY_MM.tab
private void MigrateYearlyFilesToMonthlyIfNeeded()
{
if (!_isDirectoryValid) return;
Print("Starting one-time check to migrate old yearly files (2017-2025) to monthly .tab format...");
for (int yearToMigrate = 2017; yearToMigrate <= 2025; yearToMigrate++)
{
string yearlyTabFileName = $"{Symbol.Name}_{yearToMigrate}.tab"; // Entry name if from zip
string yearlyTabFilePath = Path.Combine(_dataFolderPath, yearlyTabFileName);
string yearlyZipFilePath = Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToMigrate}.tab_zip");
// Check if monthly files for this year ALREADY exist.
bool monthlyFilesAlreadyExistForThisYear = false;
for (int m = 1; m <= 12; m++)
{
if (File.Exists(Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToMigrate}_{m:D2}.tab")) ||
File.Exists(Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToMigrate}_{m:D2}.tab_zip")))
{
monthlyFilesAlreadyExistForThisYear = true;
break;
}
}
if (monthlyFilesAlreadyExistForThisYear)
{
Print($"Monthly files for year {yearToMigrate} already exist.");
// If monthly files exist, delete any remaining original yearly file for cleanup
if (File.Exists(yearlyTabFilePath))
{
Print($"Deleting obsolete yearly file: {yearlyTabFilePath}");
try { File.Delete(yearlyTabFilePath); } catch (Exception ex) { Print($"Error deleting {yearlyTabFilePath}: {ex.Message}"); }
}
if (File.Exists(yearlyZipFilePath))
{
Print($"Deleting obsolete yearly archive: {yearlyZipFilePath}");
try { File.Delete(yearlyZipFilePath); } catch (Exception ex) { Print($"Error deleting {yearlyZipFilePath}: {ex.Message}"); }
}
continue; // Skip to next year
}
// If we reach here, no monthly files for yearToMigrate exist yet.
// Now, try to find and process the yearly source file.
List<TickDataRecord> recordsForYear = null;
string sourceYearlyFileProcessed = null; // Path of the file that was read
if (File.Exists(yearlyZipFilePath))
{
Print($"Found yearly archive {yearlyZipFilePath} for migration. Decompressing...");
sourceYearlyFileProcessed = yearlyZipFilePath;
try {
using (FileStream zipToOpen = new FileStream(yearlyZipFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (ZipArchive archive = new ZipArchive(zipToOpen, ZipArchiveMode.Read)) {
// Assume the entry name inside the zip is the original .tab filename
ZipArchiveEntry entry = archive.Entries.FirstOrDefault(e => e.Name.Equals(yearlyTabFileName, StringComparison.OrdinalIgnoreCase));
if (entry != null) {
using (Stream entryStream = entry.Open()) { recordsForYear = ReadAllRecordsFromStream(entryStream); }
} else { Print($"Entry '{yearlyTabFileName}' not found in {yearlyZipFilePath}."); }
}
} catch (Exception ex) { Print($"Error decompressing or reading {yearlyZipFilePath}: {ex.Message}"); }
}
else if (File.Exists(yearlyTabFilePath))
{
Print($"Found yearly .tab file {yearlyTabFilePath} for migration. Reading...");
sourceYearlyFileProcessed = yearlyTabFilePath;
try {
using (FileStream fs = File.OpenRead(yearlyTabFilePath)) { recordsForYear = ReadAllRecordsFromStream(fs); }
} catch (Exception ex) { Print($"Error reading {yearlyTabFilePath}: {ex.Message}"); }
}
if (recordsForYear == null || !recordsForYear.Any())
{
if (sourceYearlyFileProcessed != null) Print($"No records in {sourceYearlyFileProcessed} or read error for year {yearToMigrate}. Source file will not be deleted automatically if it was not processed.");
else Print($"No yearly file (.tab or .tab_zip) found for {yearToMigrate} to convert.");
continue; // Move to next year
}
Print($"Distributing {recordsForYear.Count} records from year {yearToMigrate} (source: {Path.GetFileName(sourceYearlyFileProcessed)}) into monthly .tab files. WARNING: This may be memory intensive.");
var monthlyGroups = recordsForYear.GroupBy(r => r.RecordTime.Month);
bool conversionSuccessAllMonths = true;
foreach (var group in monthlyGroups)
{
int month = group.Key;
string monthlyTabTargetName = $"{Symbol.Name}_{yearToMigrate}_{month:D2}.tab";
string monthlyTabTargetPath = Path.Combine(_dataFolderPath, monthlyTabTargetName);
List<TickDataRecord> monthRecords = group.OrderBy(r => r.RecordTime).ToList();
Print($"Writing {monthRecords.Count} records for {yearToMigrate}-{month:D2} to {monthlyTabTargetPath} (Mode: Create).");
if (!WriteRawCacheDataToFile(monthlyTabTargetPath, FileMode.Create, monthRecords))
{
conversionSuccessAllMonths = false;
Print($"Failed to write {monthlyTabTargetPath}. Yearly migration for {yearToMigrate} will be considered incomplete.");
break;
}
}
if (conversionSuccessAllMonths)
{
Print($"Successfully converted all months for year {yearToMigrate} from {sourceYearlyFileProcessed}. Deleting original yearly file.");
try { File.Delete(sourceYearlyFileProcessed); }
catch (Exception ex) { Print($"Error deleting original yearly file {sourceYearlyFileProcessed} after successful conversion: {ex.Message}."); }
}
else
{
Print($"Conversion of {sourceYearlyFileProcessed} for year {yearToMigrate} was incomplete due to errors. Original yearly file NOT deleted.");
}
}
Print("One-time yearly to monthly file migration check finished.");
}
private void UpdateHistoricalTabFile(int yearToUpdate, int monthToUpdate) { /* ... (as in v3.17, unchanged) ... */
if (!_isDirectoryValid) { Print($"Skipping UpdateHistoricalTabFile for {yearToUpdate}-{monthToUpdate:D2} as output directory is invalid."); return; }
Print($"Attempting to update historical .tab file for month {yearToUpdate}-{monthToUpdate:D2} in: '{_dataFolderPath}'");
string historyTabFilePath = Path.Combine(_dataFolderPath, $"{Symbol.Name}_{yearToUpdate}_{monthToUpdate:D2}.tab");
string zipFilePath = Path.ChangeExtension(historyTabFilePath, ".tab_zip");
if (File.Exists(zipFilePath)) { Print($"Skipping update for {historyTabFilePath} as a compressed version '{zipFilePath}' already exists (month likely archived)."); return; }
DateTime lastTickTimeInTabFile = GetLastTickTimeFromFile(historyTabFilePath);
if (lastTickTimeInTabFile != DateTime.MinValue) Print($"Last tick in {historyTabFilePath} is at: {lastTickTimeInTabFile:yyyy-MM-dd HH:mm:ss.fff} UTC");
else Print($"No existing .tab file found for {yearToUpdate}-{monthToUpdate:D2} or it's empty: {historyTabFilePath}");
Ticks serverTicks = MarketData.GetTicks(SymbolName);
List<TickDataRecord> recordsToAppendToTab = new List<TickDataRecord>();
DateTime earliestTickAvailableInCollection = DateTime.MaxValue;
if (serverTicks.Any()) { earliestTickAvailableInCollection = serverTicks.OrderBy(t => t.Time).First().Time; Print($"Earliest tick from server: {earliestTickAvailableInCollection:yyyy-MM-dd HH:mm:ss.fff} UTC"); }
if (lastTickTimeInTabFile != DateTime.MinValue && earliestTickAvailableInCollection > lastTickTimeInTabFile) {
Print($"Attempting to load more history for .tab. Last in file: {lastTickTimeInTabFile:yyyy-MM-dd HH:mm:ss.fff} UTC, earliest from server: {earliestTickAvailableInCollection:yyyy-MM-dd HH:mm:ss.fff} UTC");
const int maxLoadAttempts = 100;
int totalLoadedHistorically = 0;
for (int i = 0; i < maxLoadAttempts; i++) {
int newlyLoadedCount = 0;
try { newlyLoadedCount = serverTicks.LoadMoreHistory(); } catch(Exception e) { Print($"Error during LoadMoreHistory: {e.Message}"); break; }
if (newlyLoadedCount == 0 && i > 0) { Print("LoadMoreHistory returned 0 new ticks in this attempt."); break; }
totalLoadedHistorically += newlyLoadedCount;
if (!serverTicks.Any()) { Print("Ticks collection became empty after LoadMoreHistory."); break; }
earliestTickAvailableInCollection = serverTicks.OrderBy(t => t.Time).First().Time;
Print($"Total {totalLoadedHistorically} ticks potentially loaded via LoadMoreHistory. Earliest tick now: {earliestTickAvailableInCollection:yyyy-MM-dd HH:mm:ss.fff} UTC");
if (earliestTickAvailableInCollection <= lastTickTimeInTabFile) { Print("Gap to .tab file successfully bridged or covered by loaded history."); break; }
if (i == maxLoadAttempts - 1) Print("Max history load attempts reached.");
}
if (earliestTickAvailableInCollection > lastTickTimeInTabFile) {
string msg = $"Gap to .tab file for {yearToUpdate}-{monthToUpdate:D2} might still exist. Last in .tab: {lastTickTimeInTabFile:yyyy-MM-dd HH:mm:ss.fff} UTC, Earliest from server: {earliestTickAvailableInCollection:yyyy-MM-dd HH:mm:ss.fff} UTC";
Print($"WARNING: {msg}"); MessageBox.Show($"{msg}. Run backtest to rebuild history database.", $"Warning for {Symbol.Name}", MessageBoxButton.OK, MessageBoxImage.Warning, MessageBoxResult.OK);
} else if (earliestTickAvailableInCollection != DateTime.MaxValue) { Print($"History load seems to cover up to or before last tick in .tab file for {yearToUpdate}-{monthToUpdate:D2}."); }
}
var newHistoricalTicks = serverTicks.Where(t => t.Time.Year == yearToUpdate && t.Time.Month == monthToUpdate && t.Time > lastTickTimeInTabFile).OrderBy(t => t.Time);
foreach (Tick tick in newHistoricalTicks) recordsToAppendToTab.Add(new TickDataRecord(tick.Time, (float)tick.Ask, (float)tick.Bid));
if (recordsToAppendToTab.Any()) {
Print($"Found {recordsToAppendToTab.Count} new historical records to append to {historyTabFilePath}.");
try {
using (FileStream fs = new FileStream(historyTabFilePath, FileMode.Append, FileAccess.Write, FileShare.Read))
using (BinaryWriter writer = new BinaryWriter(fs)) { foreach (var record in recordsToAppendToTab) { writer.Write(record.OADateTime); writer.Write(record.Ask); writer.Write(record.Bid); } writer.Flush(); fs.Flush(true); }
Print($"Successfully appended {recordsToAppendToTab.Count} records to {historyTabFilePath}.");
} catch (Exception ex) { Print($"Error appending to historical .tab file {historyTabFilePath}: {ex.Message}"); }
} else { Print($"No new historical records found to append to .tab file for {yearToUpdate}-{monthToUpdate:D2}."); }
}
// Helper method to write a list of TickDataRecord to a specific file with a specific mode
private bool WriteRawCacheDataToFile(string filePath, FileMode mode, List<TickDataRecord> cacheToFlush) { /* ... (as in v3.17, unchanged) ... */
if (!_isDirectoryValid) { Print($"WriteRawCacheDataToFile: Skipped writing to '{filePath}' as output directory is invalid."); return false; }
if (cacheToFlush == null || !cacheToFlush.Any()) { Print($"WriteRawCacheDataToFile: Cache provided for '{filePath}' is empty or null. No data written."); return true; }
Print($"Attempting to write {cacheToFlush.Count} records to '{filePath}' using {mode}.");
try {
string fileDirectory = Path.GetDirectoryName(filePath);
if (!string.IsNullOrEmpty(fileDirectory) && !Directory.Exists(fileDirectory)) { Print($"Attempting to create directory for file: {fileDirectory}"); Directory.CreateDirectory(fileDirectory); }
using (FileStream fileStream = new FileStream(filePath, mode, FileAccess.Write, FileShare.Read))
using (BinaryWriter writer = new BinaryWriter(fileStream)) { foreach (var recordInCache in cacheToFlush) { writer.Write(recordInCache.OADateTime); writer.Write(recordInCache.Ask); writer.Write(recordInCache.Bid); } writer.Flush(); fileStream.Flush(true); }
Print($"{cacheToFlush.Count} records successfully written to '{filePath}'.");
return true;
} catch (DirectoryNotFoundException dnfe) { Print($"DIRECTORY NOT FOUND while writing to '{filePath}': {dnfe.Message}. Marking directory as invalid."); _isDirectoryValid = false; return false; }
catch (IOException ex) { Print($"IO Error writing cache to '{filePath}' using {mode}: {ex.Message}"); return false; }
catch (Exception ex) { Print($"Unexpected error writing cache to '{filePath}' using {mode}: {ex.Message}"); return false; }
}
protected override void OnStart() { /* ... (as in v3.17, with MigrateYearlyFilesToMonthlyIfNeeded call first) ... */
_tickCache = new List<TickDataRecord>();
_sessionOutputMonthsInitialized = new HashSet<string>();
_dataFolderPath = DirectoryName;
if (string.IsNullOrWhiteSpace(_dataFolderPath)) { Print("FATAL ERROR: The 'Output path' parameter is not set."); _isDirectoryValid = false; Stop(); return; }
if (!Directory.Exists(_dataFolderPath)) { Print($"FATAL ERROR: The specified Output path does not exist: '{_dataFolderPath}'. Stopping cBot."); _isDirectoryValid = false; Stop(); return; }
else { _isDirectoryValid = true; Print($"Using output directory: '{_dataFolderPath}'"); }
_systemYearAtStart = DateTime.Now.Year;
_systemMonthAtStart = DateTime.Now.Month;
_outputCurrentModeFileExtension = IsBacktesting ? ".tab" : ".tab-live";
Print("Custom binary tick data saver with caching started.");
Print($"Actual system date at start (for context): {_systemYearAtStart}-{_systemMonthAtStart:D2}");
Print("Operating Mode: {0}. File extension for output files this session: '{1}'", IsBacktesting ? "Backtesting" : "Live", _outputCurrentModeFileExtension);
if (_isDirectoryValid)
{
MigrateYearlyFilesToMonthlyIfNeeded(); // Perform one-time migration first
if (!IsBacktesting)
{
Print("Live mode: Checking for pre-existing old monthly .tab files to archive...");
for (int year = 2017; year <= _systemYearAtStart; year++) {
for (int month = 1; month <= 12; month++) {
if (year < _systemYearAtStart || (year == _systemYearAtStart && month < _systemMonthAtStart)) {
ArchivePreExistingOldTabFile(year, month);
}
}
}
Print($"Live mode: Initial update of historical .tab file for month {_systemYearAtStart}-{_systemMonthAtStart:D2}...");
UpdateHistoricalTabFile(_systemYearAtStart, _systemMonthAtStart);
Print("Live mode: Re-initializing cache for .tab-live session file.");
_tickCache.Clear(); _cachedDataYear = 0; _cachedDataMonth = 0; _sessionOutputMonthsInitialized.Clear();
Print("In live mode, current month's .tab-live data will be flushed on {0} bar close.", TimeFrame);
Print("Hourly timer starting for background updates of current month's .tab historical file.");
Timer.Start(3600);
}
} else { Print("File operations (conversion, live startup) skipped due to invalid output directory."); }
}
protected override void OnTimer() { /* ... (as in v3.17, unchanged) ... */
if (!_isDirectoryValid || IsBacktesting) return;
int currentYear = DateTime.Now.Year;
int currentMonth = DateTime.Now.Month;
Print($"Hourly timer triggered at {Server.Time:yyyy-MM-dd HH:mm:ss} UTC. Attempting to update historical .tab file for month {currentYear}-{currentMonth:D2}...");
UpdateHistoricalTabFile(currentYear, currentMonth);
}
protected override void OnBar() { /* ... (as in v3.17, unchanged) ... */
if (!_isDirectoryValid || IsBacktesting) return;
int currentProcessingYear = Server.Time.Year;
int currentProcessingMonth = Server.Time.Month;
if (_cachedDataYear == currentProcessingYear && _cachedDataMonth == currentProcessingMonth && _tickCache.Any()) {
Print($"Bar closed on {TimeFrame}. Flushing cache to '{_outputCurrentModeFileExtension}' for live month {_cachedDataYear}-{_cachedDataMonth:D2} ({_tickCache.Count} records) at {Server.Time:yyyy-MM-dd HH:mm:ss.fff} UTC.");
if (FlushOutputCacheToFile(_cachedDataYear, _cachedDataMonth)) { _tickCache.Clear(); Print("Cache successfully flushed and cleared on bar close."); }
else { Print($"Failed to write cache to '{_outputCurrentModeFileExtension}' for month {_cachedDataYear}-{_cachedDataMonth:D2} on bar close. Cache ({_tickCache.Count} records) retained.");}
}
}
protected override void OnTick() { /* ... (as in v3.17, unchanged) ... */
if (!_isDirectoryValid) return;
DateTime tickTime = Server.Time;
float askPrice = (float)Symbol.Ask;
float bidPrice = (float)Symbol.Bid;
int yearOfTick = tickTime.Year;
int monthOfTick = tickTime.Month;
var record = new TickDataRecord(tickTime, askPrice, bidPrice);
bool monthOrYearChanged = (_cachedDataYear != 0 && (_cachedDataYear != yearOfTick || _cachedDataMonth != monthOfTick));
if (monthOrYearChanged && _tickCache.Any()) {
Print($"Date change for output cache (from {_cachedDataYear}-{_cachedDataMonth:D2} to {yearOfTick}-{monthOfTick:D2}). Flushing cache for {_cachedDataYear}-{_cachedDataMonth:D2} ({_tickCache.Count} records).");
if (FlushOutputCacheToFile(_cachedDataYear, _cachedDataMonth)) _tickCache.Clear();
else { Print($"CRITICAL: Failed to flush output cache for {_cachedDataYear}-{_cachedDataMonth:D2}. Data remains in cache. Tick for {yearOfTick}-{monthOfTick:D2} dropped."); return; }
}
_cachedDataYear = yearOfTick;
_cachedDataMonth = monthOfTick;
_tickCache.Add(record);
}
private bool FlushOutputCacheToFile(int yearToFlush, int monthToFlush) { /* ... (as in v3.17, unchanged) ... */
if (!_isDirectoryValid) { Print($"Skipping FlushOutputCacheToFile for {yearToFlush}-{monthToFlush:D2}: directory invalid."); return false; }
if (!_tickCache.Any()) { Print($"FlushOutputCacheToFile called for {yearToFlush}-{monthToFlush:D2}, but cache is empty."); return true; }
string fileKey = $"{yearToFlush}{monthToFlush:D2}";
bool success = false;
if (IsBacktesting) {
string uncompressedTabFileName = $"{Symbol.Name}_{yearToFlush}_{monthToFlush:D2}.tab";
string uncompressedTabFilePath = Path.Combine(_dataFolderPath, uncompressedTabFileName);
string compressedZipFileName = $"{Symbol.Name}_{yearToFlush}_{monthToFlush:D2}.tab_zip";
string compressedZipFilePath = Path.Combine(_dataFolderPath, compressedZipFileName);
bool isOlderThanCurrentSystemMonth = yearToFlush < _systemYearAtStart || (yearToFlush == _systemYearAtStart && monthToFlush < _systemMonthAtStart);
if (isOlderThanCurrentSystemMonth) {
if (File.Exists(compressedZipFilePath)) { Print($"Backtesting: Compressed file '{compressedZipFilePath}' for older month {yearToFlush}-{monthToFlush:D2} already exists. No new file created."); success = true; }
else {
Print($"Backtesting: Generating data for older month {yearToFlush}-{monthToFlush:D2} to temporary file '{uncompressedTabFilePath}' for compression.");
if (WriteRawCacheDataToFile(uncompressedTabFilePath, FileMode.Create, _tickCache)) { CompressFileAndRemoveOriginal(uncompressedTabFilePath, compressedZipFilePath, uncompressedTabFileName); success = true; }
else { Print($"Backtesting: Failed to write .tab for {yearToFlush}-{monthToFlush:D2}. Compression skipped."); }
}
} else {
Print($"Backtesting: Generating uncompressed .tab file for current/future system month {yearToFlush}-{monthToFlush:D2}: '{uncompressedTabFilePath}'.");
success = WriteRawCacheDataToFile(uncompressedTabFilePath, FileMode.Create, _tickCache);
}
} else {
string outputFileName = $"{Symbol.Name}_{yearToFlush}_{monthToFlush:D2}{_outputCurrentModeFileExtension}";
string outputFilePath = Path.Combine(_dataFolderPath, outputFileName);
FileMode mode = _sessionOutputMonthsInitialized.Contains(fileKey) ? FileMode.Append : FileMode.Create;
Print($"Mode for output file {outputFilePath} (Live .tab-live): {mode}.");
success = WriteRawCacheDataToFile(outputFilePath, mode, _tickCache);
if (success && mode == FileMode.Create) { _sessionOutputMonthsInitialized.Add(fileKey); }
}
return success;
}
protected override void OnStop() { /* ... (as in v3.17, unchanged) ... */
Print("Stopping cBot...");
if (!IsBacktesting && _isDirectoryValid) { Timer.Stop(); Print("Hourly timer for .tab file updates stopped."); }
if (_isDirectoryValid && _tickCache.Any()) {
Print($"OnStop: Flushing remaining {_tickCache.Count} records from output cache for month {_cachedDataYear}-{_cachedDataMonth:D2} (Target ext: '{_outputCurrentModeFileExtension}').");
if (FlushOutputCacheToFile(_cachedDataYear, _cachedDataMonth)) { _tickCache.Clear(); Print("Output cache successfully flushed and cleared."); }
else { Print($"OnStop: Failed to flush remaining output cache for month {_cachedDataYear}-{_cachedDataMonth:D2}. {_tickCache.Count} records may be lost."); }
} else if (!_isDirectoryValid) { Print("OnStop: No data flushed as output directory was invalid."); }
else { Print("OnStop: No data in output cache to flush."); }
_sessionOutputMonthsInitialized.Clear();
Print("TickDataSaverCustomBinary stopped.");
}
}
}
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="cTrader.Automate" Version="*" />
</ItemGroup>
<ItemGroup>
<Reference Include="MSLib, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null">
<HintPath>..\..\..\Common\MSLib\obj\Debug\net6.0\MSLib.dll</HintPath>
</Reference>
</ItemGroup>
</Project>
@@ -0,0 +1,4 @@
// <autogenerated />
using System;
using System.Reflection;
[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v6.0", FrameworkDisplayName = "")]
@@ -0,0 +1,22 @@
//------------------------------------------------------------------------------
// <auto-generated>
// This code was generated by a tool.
//
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
using System;
using System.Reflection;
[assembly: System.Reflection.AssemblyCompanyAttribute("MS_ExportTickData")]
[assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")]
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")]
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")]
[assembly: System.Reflection.AssemblyProductAttribute("MS_ExportTickData")]
[assembly: System.Reflection.AssemblyTitleAttribute("MS_ExportTickData")]
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")]
// Generated by the MSBuild WriteCodeFragment class.
@@ -0,0 +1 @@
c3d257cf2f7ddc396be5a92ae68365f0f6daba64
@@ -0,0 +1,10 @@
is_global = true
build_property.TargetFramework = net6.0
build_property.TargetPlatformMinVersion =
build_property.UsingMicrosoftNETSdkWeb =
build_property.ProjectTypeGuids =
build_property.InvariantGlobalization =
build_property.PlatformNeutralAssembly =
build_property._SupportedPlatformList = Linux,macOS,Windows
build_property.RootNamespace = MS_ExportTickData
build_property.ProjectDir = C:\Users\Brummel\Documents\cAlgo\Sources\Robots\MS_ExportTickData\MS_ExportTickData\
@@ -0,0 +1,66 @@
{
"format": 1,
"restore": {
"C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_ExportTickData\\MS_ExportTickData\\MS_ExportTickData.csproj": {}
},
"projects": {
"C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_ExportTickData\\MS_ExportTickData\\MS_ExportTickData.csproj": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_ExportTickData\\MS_ExportTickData\\MS_ExportTickData.csproj",
"projectName": "MS_ExportTickData",
"projectPath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_ExportTickData\\MS_ExportTickData\\MS_ExportTickData.csproj",
"packagesPath": "C:\\Users\\Brummel\\.nuget\\packages\\",
"outputPath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_ExportTickData\\MS_ExportTickData\\obj\\",
"projectStyle": "PackageReference",
"configFilePaths": [
"C:\\Users\\Brummel\\AppData\\Roaming\\NuGet\\NuGet.Config"
],
"originalTargetFrameworks": [
"net6.0"
],
"sources": {
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"projectReferences": {}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
}
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"dependencies": {
"cTrader.Automate": {
"target": "Package",
"version": "[*, )"
}
},
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48"
],
"assetTargetFallback": true,
"warn": true,
"frameworkReferences": {
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\6.0.200\\RuntimeIdentifierGraph.json"
}
}
}
}
}
@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<RestoreSuccess Condition=" '$(RestoreSuccess)' == '' ">True</RestoreSuccess>
<RestoreTool Condition=" '$(RestoreTool)' == '' ">NuGet</RestoreTool>
<ProjectAssetsFile Condition=" '$(ProjectAssetsFile)' == '' ">$(MSBuildThisFileDirectory)project.assets.json</ProjectAssetsFile>
<NuGetPackageRoot Condition=" '$(NuGetPackageRoot)' == '' ">$(UserProfile)\.nuget\packages\</NuGetPackageRoot>
<NuGetPackageFolders Condition=" '$(NuGetPackageFolders)' == '' ">C:\Users\Brummel\.nuget\packages\</NuGetPackageFolders>
<NuGetProjectStyle Condition=" '$(NuGetProjectStyle)' == '' ">PackageReference</NuGetProjectStyle>
<NuGetToolVersion Condition=" '$(NuGetToolVersion)' == '' ">6.1.0</NuGetToolVersion>
</PropertyGroup>
<ItemGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<SourceRoot Include="C:\Users\Brummel\.nuget\packages\" />
</ItemGroup>
<ImportGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)ctrader.automate\1.0.12\build\cTrader.Automate.props" Condition="Exists('$(NuGetPackageRoot)ctrader.automate\1.0.12\build\cTrader.Automate.props')" />
</ImportGroup>
<PropertyGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<PkgcTrader_Automate Condition=" '$(PkgcTrader_Automate)' == '' ">C:\Users\Brummel\.nuget\packages\ctrader.automate\1.0.12</PkgcTrader_Automate>
</PropertyGroup>
</Project>
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ImportGroup Condition=" '$(ExcludeRestorePackageImports)' != 'true' ">
<Import Project="$(NuGetPackageRoot)ctrader.automate\1.0.12\build\cTrader.Automate.targets" Condition="Exists('$(NuGetPackageRoot)ctrader.automate\1.0.12\build\cTrader.Automate.targets')" />
</ImportGroup>
</Project>
@@ -0,0 +1,139 @@
{
"version": 3,
"targets": {
"net6.0": {
"cTrader.Automate/1.0.12": {
"type": "package",
"compile": {
"lib/net6.0/cAlgo.API.dll": {}
},
"runtime": {
"lib/net6.0/cAlgo.API.dll": {}
},
"build": {
"build/cTrader.Automate.props": {},
"build/cTrader.Automate.targets": {}
}
}
}
},
"libraries": {
"cTrader.Automate/1.0.12": {
"sha512": "h+HPnbqPTkoSeojoYwWWOo5Jm9ExkmrmsU/WbF/idcgO2auKAPILBghZRP6Qt5mAcE/40YCRMNHT45/kzzs5Sg==",
"type": "package",
"path": "ctrader.automate/1.0.12",
"hasTools": true,
"files": [
".nupkg.metadata",
".signature.p7s",
"build/cTrader.Automate.props",
"build/cTrader.Automate.targets",
"ctrader.automate.1.0.12.nupkg.sha512",
"ctrader.automate.nuspec",
"eula.md",
"icon.png",
"lib/net40/cAlgo.API.dll",
"lib/net40/cAlgo.API.xml",
"lib/net6.0/cAlgo.API.dll",
"lib/net6.0/cAlgo.API.xml",
"tools/net472/Core.AlgoFormat.Compose.Reflection.dll",
"tools/net472/Core.AlgoFormat.Writer.dll",
"tools/net472/Core.AlgoFormat.dll",
"tools/net472/Core.Domain.Primitives.dll",
"tools/net472/Newtonsoft.Json.dll",
"tools/net472/System.Buffers.dll",
"tools/net472/System.Collections.Immutable.dll",
"tools/net472/System.Memory.dll",
"tools/net472/System.Numerics.Vectors.dll",
"tools/net472/System.Reflection.Metadata.dll",
"tools/net472/System.Reflection.MetadataLoadContext.dll",
"tools/net472/System.Runtime.CompilerServices.Unsafe.dll",
"tools/net472/cTrader.Automate.Sdk.Tasks.dll",
"tools/net6.0/Core.AlgoFormat.Compose.Reflection.dll",
"tools/net6.0/Core.AlgoFormat.Writer.dll",
"tools/net6.0/Core.AlgoFormat.dll",
"tools/net6.0/Core.Connection.Protobuf.Common.dll",
"tools/net6.0/Core.Domain.Primitives.dll",
"tools/net6.0/Microsoft.Win32.SystemEvents.dll",
"tools/net6.0/Newtonsoft.Json.dll",
"tools/net6.0/System.Drawing.Common.dll",
"tools/net6.0/System.Reflection.MetadataLoadContext.dll",
"tools/net6.0/System.Security.Permissions.dll",
"tools/net6.0/System.Windows.Extensions.dll",
"tools/net6.0/cTrader.Automate.Sdk.Tasks.dll",
"tools/net6.0/protobuf-net.Core.dll",
"tools/net6.0/protobuf-net.dll",
"tools/net6.0/runtimes/unix/lib/net6.0/System.Drawing.Common.dll",
"tools/net6.0/runtimes/win/lib/net6.0/Microsoft.Win32.SystemEvents.dll",
"tools/net6.0/runtimes/win/lib/net6.0/System.Drawing.Common.dll",
"tools/net6.0/runtimes/win/lib/net6.0/System.Windows.Extensions.dll"
]
}
},
"projectFileDependencyGroups": {
"net6.0": [
"cTrader.Automate >= *"
]
},
"packageFolders": {
"C:\\Users\\Brummel\\.nuget\\packages\\": {}
},
"project": {
"version": "1.0.0",
"restore": {
"projectUniqueName": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_ExportTickData\\MS_ExportTickData\\MS_ExportTickData.csproj",
"projectName": "MS_ExportTickData",
"projectPath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_ExportTickData\\MS_ExportTickData\\MS_ExportTickData.csproj",
"packagesPath": "C:\\Users\\Brummel\\.nuget\\packages\\",
"outputPath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_ExportTickData\\MS_ExportTickData\\obj\\",
"projectStyle": "PackageReference",
"configFilePaths": [
"C:\\Users\\Brummel\\AppData\\Roaming\\NuGet\\NuGet.Config"
],
"originalTargetFrameworks": [
"net6.0"
],
"sources": {
"https://api.nuget.org/v3/index.json": {}
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"projectReferences": {}
}
},
"warningProperties": {
"warnAsError": [
"NU1605"
]
}
},
"frameworks": {
"net6.0": {
"targetAlias": "net6.0",
"dependencies": {
"cTrader.Automate": {
"target": "Package",
"version": "[*, )"
}
},
"imports": [
"net461",
"net462",
"net47",
"net471",
"net472",
"net48"
],
"assetTargetFallback": true,
"warn": true,
"frameworkReferences": {
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
},
"runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\6.0.200\\RuntimeIdentifierGraph.json"
}
}
}
}
@@ -0,0 +1,10 @@
{
"version": 2,
"dgSpecHash": "HudTK3f0N+tNjBJb2A2PyUN32ap0Za5MJN0GkKma6X3CtPxIznL+XMezLx70VyeNs6v0IekOGrY2sHeH8g7KIQ==",
"success": true,
"projectFilePath": "C:\\Users\\Brummel\\Documents\\cAlgo\\Sources\\Robots\\MS_ExportTickData\\MS_ExportTickData\\MS_ExportTickData.csproj",
"expectedPackageFiles": [
"C:\\Users\\Brummel\\.nuget\\packages\\ctrader.automate\\1.0.12\\ctrader.automate.1.0.12.nupkg.sha512"
],
"logs": []
}