Sean/ARM64
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/MKDT_CNN0001'
Some checks failed
Build .NET Project / build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Sean
2026-03-10 21:50:31 -04:00
parent 7d4cafafdb 7beb895ceb
commit 59b4fb0477
4 changed files with 205 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MarketData/MarketDataLib/CNNProcessing/CNNClient.cs
View File

@@ -7,7 +7,7 @@ namespace MarketData.CNNProcessing
{
public class CNNClient
{
public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,ping};
public enum Model{resnet50,resnet50B,resnet50_20241024_270,inception,vgg16,lenet5,convnext,ping};
private static readonly string Alive="Alive";
private readonly HttpClient client = new HttpClient();
private string baseUrl;

160
MarketData/MarketDataLib/CNNProcessing/DataProcessor.cs
View File

@@ -12,6 +12,7 @@ namespace MarketData.CNNProcessing
private String strFolderPath=default;
private readonly SKColor colorBlack = new SKColor(0,0,0);
private readonly SKColor colorWhite = new SKColor(255,255,255);
private readonly SKColor colorRed = new SKColor(255,0,0);
public DataProcessor(int width=640,int height=480)
{
@@ -19,6 +20,7 @@ namespace MarketData.CNNProcessing
Height=height;
PenWidth=2f;
DrawingBrush = colorBlack;
DrawingBrushRed = colorRed;
FillBrush = colorWhite;
DrawPrice=true;
UseGrayScale=false;
@@ -58,6 +60,12 @@ namespace MarketData.CNNProcessing
///<param name="value">Gets/Sets the drawing brush brush</param>
public SKColor DrawingBrush{get;set;}
/// <summary>
/// DrawingBrushRed
/// </summary>
///<param name="value">Gets/Sets the drawing brush brush</param>
public SKColor DrawingBrushRed{get;set;}
/// <summary>
/// DrawBlack
/// </summary>
@@ -142,6 +150,28 @@ namespace MarketData.CNNProcessing
if(!this.strFolderPath.EndsWith(@"/"))this.strFolderPath=this.strFolderPath+@"/";
}
/// <summary>
/// ClearFolderPath
/// </summary>
///<param name="testCases">The test cases</param>
public void ClearFolderPath()
{
if(String.IsNullOrEmpty(strFolderPath))throw new InvalidDataException($"{nameof(strFolderPath)} cannot be null");
if(!Directory.Exists(strFolderPath))
{
Directory.CreateDirectory(strFolderPath);
}
else
{
String[] pathFileNames = Directory.GetFiles(strFolderPath);
Console.WriteLine($"Deleting {pathFileNames.Length} files from {strFolderPath}");
foreach(String file in pathFileNames)
{
File.Delete(file);
}
}
}
public void ProcessData(TestCases testCases)
{
for(int index=0;index<testCases.Count;index++)
@@ -159,9 +189,15 @@ namespace MarketData.CNNProcessing
{
if(null==PenWidthArray)PenWidthArray=new float[]{PenWidth};
if(null==NoiseArray)NoiseArray=new double[]{0.00};
ProcessBollingerBandData(testCase,PenWidthArray[0],NoiseArray[0]);
}
if(testCase.TypeGenerate.Equals(TestCase.GenerateType.BollingerBand))
{
ProcessBollingerBandData(testCase,PenWidthArray[0],NoiseArray[0]);
}
else if(testCase.TypeGenerate.Equals(TestCase.GenerateType.BollingerBandWithVIX))
{
ProcessBollingerBandDataWithVolatility(testCase,PenWidthArray[0],NoiseArray[0]);
}
}
private String CreateFileName(String strFolderPath,String symbol,int dayCount,int index,int penIndex,int noiseIndex,TestCase.CaseType caseType,TestCase.GenerateType generateType,DateTime purchaseDate)
{
@@ -241,6 +277,7 @@ namespace MarketData.CNNProcessing
if(Debug || testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}");
if(null==testCase.LastPathFileName)testCase.PathFileNames.Add($"{strFolderPath}{testCase.Symbol}.jpg");
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
@@ -253,6 +290,122 @@ namespace MarketData.CNNProcessing
}
}
/// <summary>
/// ProcessBollingerBandDataWithVolatility item - Draws Price, K, L and Volatility
/// </summary>
///<param name="testCase">Symbol</param>
private void ProcessBollingerBandDataWithVolatility(TestCase testCase,float penWidth,double noise)
{
String symbolVolatility="^VIX";
DateGenerator dateGenerator=new DateGenerator();
int daysInPeriod=dateGenerator.DaysBetweenActual(testCase.PurchaseDate,testCase.HistDate);
daysInPeriod+=60;
Prices prices=PricingDA.GetPrices(testCase.Symbol,testCase.PurchaseDate,daysInPeriod);
Prices volatilityPrices=PricingDA.GetPrices(symbolVolatility,testCase.PurchaseDate,daysInPeriod);
BollingerBands bollingerBands=BollingerBandGenerator.GenerateBollingerBands(prices); // we want to grab K, L, and Close
bollingerBands=new BollingerBands(bollingerBands.Where(x=>x.Date>=testCase.HistDate).ToList());
float[] k=new float[bollingerBands.Count];
float[] l=new float[bollingerBands.Count];
float[] close=new float[bollingerBands.Count];
// Line up volatility dates with bollinger bands
DateTime minDate = bollingerBands.Min(x=>x.Date);
DateTime maxDate = bollingerBands.Max(x=>x.Date);
volatilityPrices = new Prices(volatilityPrices.Where(x=>x.Date<=maxDate && x.Date>=minDate).OrderBy(x=>x.Date).ToList()); // most historical date in lowest index
float[] v=volatilityPrices.GetPrices();
float minV=Numerics.Min(ref v); // get the minimum volatility value
double minP=bollingerBands.Min(x=>x.Close); // get minimum price
double factor=minP/minV; // determine scaling factor
for(int index=0;index<v.Length;index++)
{
double item = v[index];
item*=factor;
v[index]=(float)Math.Log(item)*1000.00f;
}
// populate the arrays in reverse order so that we have the most historical date in the lowest index
for(int index=bollingerBands.Count-1;index>=0;index--)
{
BollingerBandElement bollingerBandElement=bollingerBands[index];
k[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.K)*1000.00f; // put the data in log form
l[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.L)*1000.00f; // put the data in log form
close[bollingerBands.Count-index-1]=(float)Math.Log(bollingerBandElement.Close)*1000.00f; // put the data in log form
}
Numerics.ZeroForNaNOrInfinity(ref k);
Numerics.ZeroForNaNOrInfinity(ref l);
Numerics.ZeroForNaNOrInfinity(ref close);
Numerics.ZeroForNaNOrInfinity(ref v);
float maxY=Math.Max(Math.Max(Numerics.Max(ref l),Math.Max(Numerics.Max(ref close),Numerics.Max(ref k))),Numerics.Max(ref v));
float minY=Math.Min(Math.Min(Numerics.Min(ref l),Math.Min(Numerics.Min(ref close),Numerics.Min(ref k))),Numerics.Min(ref v))-5f;
float maxX=close.Length;
float minX=0.00f;
// Pen pen=new Pen(DrawingBrush,penWidth);
// Pen redPen=new Pen(DrawingBrushRed,penWidth);
ImageHelper imageHelper=new ImageHelper();
PointMapping pointMapping=new PointMapping(Width,Height,maxX,minX,maxY,minY);
imageHelper.CreateImage(Width,Height,pointMapping);
imageHelper.Fill(FillBrush);
LineSegments lineSegments=new LineSegments();
// draw volatility
for(int index=0;index<v.Length;index++)
{
if(0==index)continue;
SKPoint p1=new SKPoint(index-1,(int)v[index-1]);
SKPoint p2=new SKPoint(index,(int)v[index]);
lineSegments.Add(p1,p2);
}
imageHelper.DrawPath(DrawingBrushRed,penWidth,lineSegments);
// draw prices
lineSegments.Clear();
for(int index=0;index<close.Length && DrawPrice;index++)
{
if(0==index)continue;
SKPoint p1=new SKPoint(index-1,(int)close[index-1]);
SKPoint p2=new SKPoint(index,(int)close[index]);
lineSegments.Add(p1,p2);
}
imageHelper.DrawPath(DrawingBrush,penWidth,lineSegments);
// draw k
lineSegments.Clear();
for(int index=0;index<k.Length;index++)
{
if(0==index)continue;
SKPoint p1=new SKPoint(index-1,(int)k[index-1]);
SKPoint p2=new SKPoint(index,(int)k[index]);
lineSegments.Add(p1,p2);
}
imageHelper.DrawPath(DrawingBrush,penWidth,lineSegments);
// draw l
lineSegments.Clear();
for(int index=0;index<l.Length;index++)
{
if(0==index)continue;
SKPoint p1=new SKPoint(index-1,(int)l[index-1]);
SKPoint p2=new SKPoint(index,(int)l[index]);
lineSegments.Add(p1,p2);
}
imageHelper.DrawPath(DrawingBrush,penWidth,lineSegments);
if(0.00!=noise)imageHelper.AddNoise(NoiseColor,noise);
if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}");
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
else imageHelper.Save(testCase.LastPathFileName);
}
else
{
testCase.Streams.Add(imageHelper.ToStream());
}
}
/// <summary>
/// ProcessPriceData item
/// </summary>
@@ -292,6 +445,7 @@ namespace MarketData.CNNProcessing
if(testCase.TypeOutput.Equals(TestCase.OutputType.OutputFile))
{
MDTrace.WriteLine(LogLevel.DEBUG,$"Writing {testCase.LastPathFileName}");
if(File.Exists(testCase.LastPathFileName))File.Delete(testCase.LastPathFileName);
if(UseGrayScale)imageHelper.SaveGrayScaleJPG(testCase.LastPathFileName);
else imageHelper.SaveBlackAndWhiteJPG(testCase.LastPathFileName);

2
MarketData/MarketDataLib/CNNProcessing/TestCase.cs
View File

@@ -14,7 +14,7 @@ namespace MarketData.CNNProcessing
public class TestCase
{
public enum CaseType{Training,Test,Validation};
public enum GenerateType{Price,BollingerBand};
public enum GenerateType{Price,BollingerBand,BollingerBandWithVIX};
public enum OutputType{OutputFile,OutputStream}
private readonly List<Stream> streams=new List<Stream>();
private readonly List<String> pathFileNames=new List<String>();

51
MarketData/MarketDataLib/Generator/CMMomentum/CMMomentumGenerator.cs
View File

@@ -129,19 +129,31 @@ namespace MarketData.Generator.CMMomentum
}
return true;
}
// This method is made public in order that it can be tested
/// <summary>
/// PredictCandidate - 2026 convnext version - Be sure that the model parameters has 90 days in UseCNNDayCount
/// If we need to revert back to CNNClient.Model.resnet50_20241024_270 then uncomment the below method and update cmParams.UseCNNDayCount=270
/// </summary>
/// <param name="cmCandidate"></param>
/// <param name="cmParams"></param>
/// <returns></returns>
public static bool PredictCandidate(CMCandidate cmCandidate,CMParams cmParams)
{
try
{
CNNClient cnnClient=new CNNClient(cmParams.UseCNNHost);
DataProcessor dataProcessor=new DataProcessor();
dataProcessor.Width=128;
dataProcessor.Height=128;
int imageDimensions=224;
dataProcessor.Width=imageDimensions;
dataProcessor.Height=imageDimensions;
dataProcessor.PenWidth=1;
TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,TestCase.OutputType.OutputStream);
if(90!=cmParams.UseCNNDayCount)
{
throw new InvalidDataException("CNNClient.Model.convnext must be used with cmParams.UseCNNDayCount=90.");
}
TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBandWithVIX,TestCase.OutputType.OutputStream);
dataProcessor.ProcessData(testCase);
String prediction = cnnClient.Predict(CNNClient.Model.resnet50_20241024_270,testCase.LastStream);
Stream streamResult = cnnClient.ProcessImage(testCase.LastStream); // process the image through PIL
String prediction = cnnClient.Predict(CNNClient.Model.convnext,streamResult);
prediction=prediction.Substring(prediction.IndexOf("-->"));
int result=int.Parse(Utility.BetweenString(prediction,"[[","]"));
if(1==result)
@@ -157,5 +169,34 @@ namespace MarketData.Generator.CMMomentum
return false;
}
}
// Keep this until happy with the new model
// This method is made public in order that it can be tested
// public static bool PredictCandidate(CMCandidate cmCandidate,CMParams cmParams)
// {
// try
// {
// CNNClient cnnClient=new CNNClient(cmParams.UseCNNHost);
// DataProcessor dataProcessor=new DataProcessor();
// dataProcessor.Width=128;
// dataProcessor.Height=128;
// dataProcessor.PenWidth=1;
// TestCase testCase=new TestCase(cmCandidate.Symbol,cmCandidate.TradeDate,cmParams.UseCNNDayCount,TestCase.CaseType.Test,TestCase.GenerateType.BollingerBand,TestCase.OutputType.OutputStream);
// dataProcessor.ProcessData(testCase);
// String prediction = cnnClient.Predict(CNNClient.Model.resnet50_20241024_270,testCase.LastStream);
// prediction=prediction.Substring(prediction.IndexOf("-->"));
// int result=int.Parse(Utility.BetweenString(prediction,"[[","]"));
// if(1==result)
// {
// cmCandidate.Score*=(1.00+cmParams.UseCNNRewardPercentDecimal); // increase the score by the percentage indicated in the params settings
// cmCandidate.CNNPrediction=true;
// }
// return true;
// }
// catch(Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Error encountered calling convolutional model at {0}. Exception was {1}",cmParams.UseCNNHost,exception.ToString()));
// return false;
// }
// }
}
}