I have a problem concerning a complex JSON deserialization in c# using newtonsoft.json
1- my feed provider send me a gz file push through POST request each 3 seconds (size between 200ko and 3000 ko)
2- I get it back, unzip it and download the JSON file inside on my server
3- I need to treat this JSON file directly after receiving it (so in realtime). What I do is using a "File System watcher" class included into a timed background task (using "TimedHostedService" class)
When a json file is created into "tmp" folder, the file system watcher detect the change and I can apply a deserialization of this json file just after.
The result until now is a deserialization which works for a few moments, stopped for few moments, and working for few moments, etc. ... Sometimes it put the data in database, sometimes not => there's a problem of latency (after some debug, I isolated the problem and it seems to be a memory issue or maybe needs more time to treat than 3 seconds push interval. there's a latency when apply this line of code : Goalserve objects = serializer.Deserialize(reader);)
My app is a .net core 2.0 app. this is all parts of the code:
Startup.cs
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using System;
using System.IO;
using System.Net;
using static HelloWorld.Controllers.ValuesController;
namespace HelloWorld
{
public class Startup
{
// This method gets called by the runtime. Use this method to add services to the container.
// For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940
public void ConfigureServices(IServiceCollection services)
{
services.AddMvc();
services.AddHostedService<TimedHostedService>();
/*
services.Configure<IISServerOptions>(options =>
{
options.AutomaticAuthentication = false;
});
*/
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IHostingEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
app.UseMvc();
app.Run(async (context) =>
{
await context.Response.WriteAsync("DEV Testing environment");
});
}
}
}
Goalserve class model
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;
namespace HelloWorld.Models
{
public partial class Goalserve
{
[JsonProperty("updated")]
public string Updated { get; set; }
[JsonProperty("updated_ts")]
public long UpdatedTs { get; set; }
[JsonProperty("events")]
public Dictionary<string, MatchIdDetails> Events { get; set; }
}
public partial class MatchIdDetails
{
[JsonProperty("core")]
public Core Core { get; set; }
[JsonProperty("info")]
public InfoClass Info { get; set; }
[JsonProperty("stats")]
public Dictionary<string, Stat> Stats { get; set; }
[JsonProperty("odds")]
public Dictionary<string, Odd> Odds { get; set; }
}
public partial class Core
{
[JsonProperty("safe")]
public long Safe { get; set; }
[JsonProperty("stopped")]
public long Stopped { get; set; }
[JsonProperty("blocked")]
public long Blocked { get; set; }
[JsonProperty("finished")]
public long Finished { get; set; }
[JsonProperty("updated")]
public DateTimeOffset Updated { get; set; }
[JsonProperty("updated_ts")]
public long UpdatedTs { get; set; }
}
public partial class InfoClass
{
[JsonProperty("id")]
public long Id { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("sport")]
public string Sport { get; set; }
[JsonProperty("league")]
public string League { get; set; }
[JsonProperty("start_time")]
public string StartTime { get; set; }
[JsonProperty("start_date")]
public string StartDate { get; set; }
[JsonProperty("start_ts")]
public long StartTs { get; set; }
[JsonProperty("period")]
public string Period { get; set; }
[JsonConverter(typeof(ParseStringConverter))]
[JsonProperty("minute")]
public long Minute { get; set; }
[JsonProperty("secunds")]
public string Secunds { get; set; }
[JsonProperty("score")]
public string Score { get; set; }
[JsonProperty("points")]
public string Points { get; set; }
[JsonProperty("pitch")]
public string Pitch { get; set; }
[JsonProperty("ball_pos")]
public string BallPos { get; set; }
[JsonProperty("add_time")]
public string AddTime { get; set; }
[JsonProperty("player")]
public string Player { get; set; }
[JsonConverter(typeof(ParseStringConverter))]
[JsonProperty("state")]
public long State { get; set; }
}
public partial class Odd
{
[JsonProperty("id")]
public long Id { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("short_name")]
public string ShortName { get; set; }
[JsonProperty("suspend")]
public long Suspend { get; set; }
[JsonProperty("order")]
public long Order { get; set; }
[JsonProperty("info")]
[JsonConverter(typeof(InfoEnumConverter))]
public InfoEnum Info { get; set; }
[JsonProperty("participants")]
public Dictionary<string, Participant> Participants { get; set; }
}
public partial class Participant
{
[JsonProperty("id")]
public long Id { get; set; }
[JsonProperty("order")]
public long Order { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("short_name")]
public string ShortName { get; set; }
[JsonProperty("value_eu")]
public string ValueEu { get; set; }
[JsonProperty("value_na")]
public string ValueNa { get; set; }
[JsonProperty("value_us")]
public string ValueUs { get; set; }
[JsonProperty("handicap")]
public string Handicap { get; set; }
[JsonProperty("suspend")]
public long Suspend { get; set; }
}
public partial class Stat
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("home")]
public string Home { get; set; }
[JsonProperty("away")]
public string Away { get; set; }
}
public enum InfoEnum { Count070007959, CurrentCorners11, Empty, OtherType };
internal static class Converter
{
public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings
{
MetadataPropertyHandling = MetadataPropertyHandling.Ignore,
DateParseHandling = DateParseHandling.None,
ContractResolver = new CamelCasePropertyNamesContractResolver(),
Converters =
{
InfoEnumConverter.Singleton,
ParseStringConverter.Singleton,
new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal }
},
};
}
public class ParseStringConverter : JsonConverter
{
public override bool CanConvert(Type t) => t == typeof(long) || t == typeof(long?);
public override object ReadJson(JsonReader reader, Type t, object existingValue, JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null) return null;
var value = serializer.Deserialize<string>(reader);
long l;
if (Int64.TryParse(value, out l))
{
return l;
}
throw new Exception("Cannot unmarshal type long");
}
public override void WriteJson(JsonWriter writer, object untypedValue, JsonSerializer serializer)
{
if (untypedValue == null)
{
serializer.Serialize(writer, null);
return;
}
var value = (long)untypedValue;
serializer.Serialize(writer, value.ToString());
return;
}
public static readonly ParseStringConverter Singleton = new ParseStringConverter();
}
public class InfoEnumConverter : JsonConverter
{
public override bool CanConvert(Type t) => t == typeof(InfoEnum) || t == typeof(InfoEnum?);
public override object ReadJson(JsonReader reader, Type t, object existingValue, JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null) return null;
var value = serializer.Deserialize<string>(reader);
switch (value)
{
case "":
return InfoEnum.Empty;
case "Count : 0 (70:00 - 79:59)":
return InfoEnum.Count070007959;
case "Current Corners : 10":
return InfoEnum.CurrentCorners11;
default:
return InfoEnum.OtherType;
}
//throw new Exception("Cannot unmarshal type InfoEnum");
}
public override void WriteJson(JsonWriter writer, object untypedValue, JsonSerializer serializer)
{
if (untypedValue == null)
{
serializer.Serialize(writer, null);
return;
}
var value = (InfoEnum)untypedValue;
switch (value)
{
case InfoEnum.Empty:
serializer.Serialize(writer, "");
return;
case InfoEnum.Count070007959:
serializer.Serialize(writer, "Count : 0 (70:00 - 79:59)");
return;
case InfoEnum.CurrentCorners11:
serializer.Serialize(writer, "Current Corners : 11");
return;
}
throw new Exception("Cannot marshal type InfoEnum");
}
public static readonly InfoEnumConverter Singleton = new InfoEnumConverter();
}
}
This is an example of JSON feed received from my feed provider : https://filebin.net/k5enw2wn4f5bc89m/inplay-soccer.json?t=84y0df94
ValuesController
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using HelloWorld.Models;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
// For more information on enabling Web API for empty projects, visit https://go.microsoft.com/fwlink/?LinkID=397860
namespace HelloWorld.Controllers
{
[Route("")]
public class ValuesController : Controller
{
// POST api/<controller>
[HttpPost]
[Consumes("application/gzip")]
public async Task PostAsync()
{
var timestamp = new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds();
WebClient Client = new WebClient();
Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\inplay-soccer_" + timestamp + ".gz");
using (var inputFileStream = new FileStream("c:\\temp\\inplay-soccer_" + timestamp + ".gz", FileMode.Open))
using (var gzipStream = new GZipStream(inputFileStream, CompressionMode.Decompress))
using (var outputFileStream = new FileStream("c:\\temp_json\\inplay-soccer_" + timestamp + ".json", FileMode.Create))
{
await gzipStream.CopyToAsync(outputFileStream);
}
}
internal class TimedHostedService : IHostedService, IDisposable
{
private readonly ILogger _logger;
private Timer _timer;
FileProcessor fileProcessor = new FileProcessor();
FileSystemWatcher watcher = new FileSystemWatcher();
public TimedHostedService(ILogger<TimedHostedService> logger)
{
_logger = logger;
}
public Task StartAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Timed Background Service is starting.");
var startTimeSpan = TimeSpan.Zero;
var periodTimeSpan = TimeSpan.FromSeconds(1);
watcher.Path = "c:\\temp_json\\";
watcher.Filter = "*.json";
watcher.Created += OnChanged;
watcher.Changed += OnChanged;
watcher.EnableRaisingEvents = true;
return Task.CompletedTask;
}
private void OnChanged(object sender, FileSystemEventArgs e)
{
//WebClient Client = new WebClient();
//Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\inplay-soccer-jecoute-toutes-les-1sec"+e.Name+");
if (e.ChangeType == WatcherChangeTypes.Created)
{
var str = e.Name;
str = str.Remove(str.Length - 5);
//Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\"+str);
// ...enqueue it's file name so it can be processed...
fileProcessor.EnqueueFileName(str);
}
}
// File processor class
class FileProcessor : IDisposable
{
// Create an AutoResetEvent EventWaitHandle
private EventWaitHandle eventWaitHandle = new AutoResetEvent(false);
private Thread worker;
private readonly object locker = new object();
private Queue<string> fileNamesQueue = new Queue<string>();
public FileProcessor()
{
// Create worker thread
worker = new Thread(Work);
// Start worker thread
worker.Start();
}
public void EnqueueFileName(string FileName)
{
// Enqueue the file name
// This statement is secured by lock to prevent other thread to mess with queue while enqueuing file name
lock (locker) fileNamesQueue.Enqueue(FileName);
// Signal worker that file name is enqueued and that it can be processed
eventWaitHandle.Set();
}
private void Work()
{
while (true)
{
string fileName = null;
// Dequeue the file name
lock (locker)
if (fileNamesQueue.Count > 0)
{
fileName = fileNamesQueue.Dequeue();
// If file name is null then stop worker thread
if (fileName == null) return;
}
if (fileName != null)
{
// Process file
ProcessFile(fileName);
}
else
{
// No more file names - wait for a signal
eventWaitHandle.WaitOne();
}
}
}
private void ProcessFile(string FileName)
{
PutInDb(FileName);
// Maybe it has to wait for file to stop being used by process that created it before it can continue
// Read content file
// Log file data to database
// Move file to archive folder
}
#region IDisposable Members
public void Dispose()
{
// Signal the FileProcessor to exit
EnqueueFileName(null);
// Wait for the FileProcessor's thread to finish
worker.Join();
eventWaitHandle.Close();
}
#endregion
}
private void DoWork(object state)
{
_logger.LogInformation("Timed Background Service is working.");
WebClient Client = new WebClient();
Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\inplay-soccer-jecoute-toutes-les-1sec.gz");
}
public Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("Timed Background Service is stopping.");
_timer?.Change(Timeout.Infinite, 0);
return Task.CompletedTask;
}
public void Dispose()
{
_timer?.Dispose();
}
}
public static void PutInDb(string save)
{
WebClient Client = new WebClient();
using (StreamReader r = new StreamReader("c:\\temp_json\\" + save + ".json"))
{
var con = new SqlConnection(@"Server=WINDOWS-ASPNET-\SQLEXPRESS01;Database=goalserve;Trusted_Connection=false;User ID=******;Password=*********;");
try
{
con.Open();
}
catch
{
//Client.DownloadFile("http://inplay.goalserve.com/inplay-soccer.gz", "C:\\temp\\inplay-soccer_noooo" + save + ".gz");
}
var cmd = new SqlCommand("INSERT INTO Event (id, league, name, start_date, start_time, start_ts, score, period, minute, seconds, additional_time, state, ball_pos, player) VALUES (@id, @league, @name, @start_date, @start_time, @start_ts, @score, @period, @minute, @seconds, @additional_time, @state, @ball_pos, @player)", con);
using (JsonTextReader reader = new JsonTextReader(r))
{
reader.SupportMultipleContent = true;
var serializer = new JsonSerializer();
while (reader.Read())
{
if (reader.TokenType == JsonToken.StartObject)
{
Goalserve objects = serializer.Deserialize<Goalserve>(reader);
var data = objects.Events.Keys.ToArray();
for (int i = 0; i < data.Count(); i++)
{
// just test all this treatment with one random match data
if (objects.Events[data[i].ToString()].Info.Name == "America RJ vs Macae Esporte FC")
{
cmd.Parameters.Add("@id", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Id;
cmd.Parameters.Add("@league", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.League;
cmd.Parameters.Add("@name", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Name;
cmd.Parameters.Add("@start_date", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartDate;
cmd.Parameters.Add("@start_time", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartTime;
cmd.Parameters.Add("@start_ts", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartTs;
cmd.Parameters.Add("@score", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Score;
cmd.Parameters.Add("@period", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Period;
cmd.Parameters.Add("@minute", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Minute;
cmd.Parameters.Add("@seconds", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Secunds;
cmd.Parameters.Add("@additional_time", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.AddTime;
cmd.Parameters.Add("@state", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.State;
cmd.Parameters.Add("@ball_pos", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.BallPos;
cmd.Parameters.Add("@player", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Player;
cmd.ExecuteNonQuery();
}
}
}
}
}
}
}
}
}
Please give me your own implementation even if I need to change all the method initially used.
thank you in advance !
EDIT2 ValuesController
[HttpPost]
[Consumes("application/gzip")]
public async Task PostAsync()
{
var con = new SqlConnection(@"Server=WINDOWS-ASPNET-\SQLEXPRESS01;Database=goalserve;Trusted_Connection=false;User ID=*****;Password=*****;");
var cmd = new SqlCommand("INSERT INTO Event (id, league, name, start_date, start_time, start_ts, score, period, minute, seconds, additional_time, state, ball_pos, player) VALUES (@id, @league, @name, @start_date, @start_time, @start_ts, @score, @period, @minute, @seconds, @additional_time, @state, @ball_pos, @player)", con);
var timestamp = new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds();
var client = new HttpClient();
var response = await client.GetAsync(@"http://inplay.goalserve.com/inplay-soccer.gz");
using (var stream = await response.Content.ReadAsStreamAsync())
{
var fileInfo = new FileInfo("C:\\temp\\inplay - soccer_"+timestamp+".gz");
using (var fileStream = fileInfo.OpenWrite())
{
await stream.CopyToAsync(fileStream);
}
}
using (FileStream fInStream = new FileStream(@"C:\\temp\\inplay - soccer_" + timestamp + ".gz", FileMode.Open, FileAccess.Read))
{
using (GZipStream zipStream = new GZipStream(fInStream, CompressionMode.Decompress))
{
using (FileStream fOutStream = new FileStream(@"C:\\temp\\inplay - soccer_" + timestamp + ".json", FileMode.Create, FileAccess.Write))
{
byte[] tempBytes = new byte[4096];
int i;
while ((i = zipStream.Read(tempBytes, 0, tempBytes.Length)) != 0)
{
fOutStream.Write(tempBytes, 0, i);
}
}
}
}
using (StreamReader r = new StreamReader("c:\\temp\\inplay - soccer_" + timestamp + ".json"))
{
var objects = new Goalserve();
string json = r.ReadToEnd();
objects = JsonConvert.DeserializeObject<Goalserve>(json);
var data = objects.Events.Keys.ToArray();
for (int i = 0; i < data.Count(); i++)
{
if (objects.Events[data[i].ToString()].Info.Name == "Blackburn vs Wigan")
{
con.Open();
cmd.Parameters.Add("@id", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Id;
cmd.Parameters.Add("@league", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.League;
cmd.Parameters.Add("@name", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Name;
cmd.Parameters.Add("@start_date", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartDate;
cmd.Parameters.Add("@start_time", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartTime;
cmd.Parameters.Add("@start_ts", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.StartTs;
cmd.Parameters.Add("@score", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Score;
cmd.Parameters.Add("@period", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Period;
cmd.Parameters.Add("@minute", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Minute;
cmd.Parameters.Add("@seconds", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Secunds;
cmd.Parameters.Add("@additional_time", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.AddTime;
cmd.Parameters.Add("@state", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.State;
cmd.Parameters.Add("@ball_pos", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.BallPos;
cmd.Parameters.Add("@player", SqlDbType.VarChar).Value = objects.Events[data[i].ToString()].Info.Player;
cmd.ExecuteNonQuery();
}
}
Taskis always prefered as it works on the thread pool and the threads get managed by thread pool. UsingWebClientis pretty much deprecated too, tool of choice should beHttpClientand IHttpClientFactory and the like (named/typed clients). Why are you not downloading, unzipping and inserting as a stream w/o saving to file system first?GZipStreamand pass that to aStreamReader/JsonReaderor just directly toJsonReader. Then the file will be parsed as its downloaded. No file system invovled. Threads not necessary either as you can do everything via asyc Task