using Microsoft.Extensions.Configuration; using System; using System.Data.SqlClient; using Dapper; using Dapper.Contrib; using PornocopiaVisionMetadataExtractor.Data; using System.Linq; using System.Reflection; using System.Collections.Generic; using System.Transactions; using Dapper.Contrib.Extensions; using System.Net.Http; using System.Text.Json; namespace PornocopiaVisionMetadataExtractor { class Program { static void Main(string[] args) { var configurationBuilder = new ConfigurationBuilder(); configurationBuilder.SetBasePath(System.IO.Directory.GetCurrentDirectory()); // errors here configurationBuilder.AddJsonFile(path: "appSettings.json", optional: false, reloadOnChange: true); // errors here //configurationBuilder.AddXmlFile(Assembly.GetExecutingAssembly().Location + ".config", optional: false, reloadOnChange: true); var config = configurationBuilder.Build(); DateTime lastRun = DateTime.Parse("2021-01-01"); try { String lastRuntime = System.IO.File.ReadAllText("lastrun"); DateTime.TryParse(lastRuntime, out lastRun); } catch (Exception) { } List posts; DateTime? newestPost = null; int page = 0; int requestID = 1; String connectionString = config.GetConnectionString("Pornocopia"); String baseImageServerURL = config.GetSection("AppSettings")["BaseImageServerURL"]; String VisionMetadataURL = config.GetSection("AppSettings")["VisionMetadataURL"]; do { using (SqlConnection connection = new SqlConnection(connectionString)) { connection.Open(); var storedProcedureParameters = new DynamicParameters(); storedProcedureParameters.Add("pageSize", 100); //temp dont increment because new posts will be filtered offsetting the page.... storedProcedureParameters.Add("currentPage", page++); using (var multi = connection.QueryMultiple("GetMetadataMissingForumPosts", storedProcedureParameters, commandType: System.Data.CommandType.StoredProcedure)) { posts = multi.Read().ToList(); var images = multi.Read(); foreach (var post in posts) post.Images = images.Where(img => img.ForumPostID == post.ForumPostID).ToList(); } } if (!newestPost.HasValue) newestPost = posts.Max(posts => posts.Timestamp); var http = new System.Net.WebClient(); //Download image using (SqlConnection connection = new SqlConnection(connectionString)) { connection.Open(); foreach (var post in posts) { using (var transaction = connection.BeginTransaction()) { List bmiValues = new List(); List ageValues = new List(); try { foreach (var image in post.Images) { try { String downloadURL = $"{baseImageServerURL}/{image.ImageLocation}"; byte[] data = http.DownloadData(downloadURL); String submissionFilename = image.ImageLocation.Substring(image.ImageLocation.LastIndexOf('/') + 1); //Do vision work var httpVisionClient = new HttpClient(); MultipartFormDataContent form = new MultipartFormDataContent(); form.Add(new StringContent(requestID.ToString()), "request_id"); form.Add(new ByteArrayContent(data, 0, data.Length), "image_file", submissionFilename); requestID++; var httpTask = httpVisionClient.PostAsync(VisionMetadataURL, form); httpTask.Wait(); var response = httpTask.Result; var StringReadTask = response.Content.ReadAsStringAsync(); StringReadTask.Wait(); var responseString = StringReadTask.Result; if (responseString.Contains("500 Internal Server Error")) continue; var jsonOptions = new JsonSerializerOptions() { PropertyNameCaseInsensitive = true }; var cvResponse = JsonSerializer.Deserialize(responseString, jsonOptions); if (cvResponse.Results.Count() > 0) { bmiValues.AddRange(cvResponse.Results.Select(a => a.BMI)); ageValues.AddRange(cvResponse.Results.Select(a => a.Age)); } //JsonDocument doc = JsonDocument.Parse(responseString); //var jsonResponse = doc.RootElement; //var bmi = jsonResponse.GetProperty("bmi").GetDecimal(); //if (bmi > 0) // bmiValues.Add(bmi); System.Threading.Thread.Sleep(1000); } catch(Exception ex) { Console.WriteLine($"Warn: {ex.ToString()}"); } } if (bmiValues.Count == 0 || ageValues.Count == 0) continue; var ageMetadata = new ImageMetadata() { ForumPostID = post.ForumPostID, Name = "Age", ValueDecimal = ageValues.Average() }; connection.Insert(ageMetadata, transaction); var bmiMetadata = new ImageMetadata() { ForumPostID = post.ForumPostID, Name = "BMI", ValueDecimal = bmiValues.Average() }; connection.Insert(bmiMetadata, transaction); transaction.Commit(); Console.WriteLine($"Commited age: {ageValues.Average()} and BMI: {bmiValues.Average()} for ForumPostID: {post.ForumPostID}"); } catch (Exception ex) { } } } } } while (posts.Count == 0 || posts.Min(posts => posts.Timestamp) > lastRun); System.IO.File.WriteAllText("lastrun", lastRun.ToString()); Console.WriteLine("done"); } } }