PVisionMetadataExtractor/PornocopiaVisionMetadataExtractor/Program.cs

using Microsoft.Extensions.Configuration;
using System;
using System.Data.SqlClient;
using Dapper;
using Dapper.Contrib;
using PornocopiaVisionMetadataExtractor.Data;
using System.Linq;
using System.Reflection;
using System.Collections.Generic;
using System.Transactions;
using Dapper.Contrib.Extensions;
using System.Net.Http;
using System.Text.Json;


namespace PornocopiaVisionMetadataExtractor
{
    class Program
    {
        static void Main(string[] args)
        {
            var configurationBuilder = new ConfigurationBuilder();

            configurationBuilder.SetBasePath(System.IO.Directory.GetCurrentDirectory());  // errors here
            configurationBuilder.AddJsonFile(path: "appSettings.json", optional: false, reloadOnChange: true); // errors here
            //configurationBuilder.AddXmlFile(Assembly.GetExecutingAssembly().Location + ".config", optional: false, reloadOnChange: true);
            var config = configurationBuilder.Build();

            DateTime lastRun = DateTime.Parse("2021-01-01");
            try
            {
                String lastRuntime = System.IO.File.ReadAllText("lastrun");
                DateTime.TryParse(lastRuntime, out lastRun);
            }
            catch (Exception) { }

            List<ForumPost> posts;
            DateTime? newestPost = null;
            int page = 0;
            int requestID = 1;
            String connectionString = config.GetConnectionString("Pornocopia");
            String baseImageServerURL = config.GetSection("AppSettings")["BaseImageServerURL"];
            String VisionMetadataURL = config.GetSection("AppSettings")["VisionMetadataURL"];
            do
            {
                using (SqlConnection connection = new SqlConnection(connectionString))
                {
                    connection.Open();

                    var storedProcedureParameters = new DynamicParameters();
                    storedProcedureParameters.Add("pageSize", 100);
                    //temp dont increment because new posts will be filtered offsetting the page....
                    storedProcedureParameters.Add("currentPage", page++);

                    using (var multi = connection.QueryMultiple("GetMetadataMissingForumPosts", storedProcedureParameters, commandType: System.Data.CommandType.StoredProcedure))
                    {
                        posts = multi.Read<ForumPost>().ToList();
                        var images = multi.Read<ForumPostImage>();
                        foreach (var post in posts)
                            post.Images = images.Where(img => img.ForumPostID == post.ForumPostID).ToList();
                    }
                }
                if (!newestPost.HasValue)
                    newestPost = posts.Max(posts => posts.Timestamp);
                var http = new System.Net.WebClient();
                //Download image
                using (SqlConnection connection = new SqlConnection(connectionString))
                {
                    connection.Open();
                    foreach (var post in posts)
                    {
                        using (var transaction = connection.BeginTransaction())
                        {
                            List<Decimal> bmiValues = new List<decimal>();
                            List<Decimal> ageValues = new List<decimal>();

                            try
                            {
                                foreach (var image in post.Images)
                                {
                                    try
                                    {
                                        String downloadURL = $"{baseImageServerURL}/{image.ImageLocation}";
                                        byte[] data = http.DownloadData(downloadURL);
                                        String submissionFilename = image.ImageLocation.Substring(image.ImageLocation.LastIndexOf('/') + 1);
                                        //Do vision work
                                        var httpVisionClient = new HttpClient();
                                        MultipartFormDataContent form = new MultipartFormDataContent();
                                        form.Add(new StringContent(requestID.ToString()), "request_id");
                                        form.Add(new ByteArrayContent(data, 0, data.Length), "image_file", submissionFilename);
                                        requestID++;

                                        var httpTask = httpVisionClient.PostAsync(VisionMetadataURL, form);
                                        httpTask.Wait();
                                        var response = httpTask.Result;

                                        var StringReadTask = response.Content.ReadAsStringAsync();
                                        StringReadTask.Wait();
                                        var responseString = StringReadTask.Result;
                                        if (responseString.Contains("500 Internal Server Error"))
                                            continue;
                                        var jsonOptions = new JsonSerializerOptions() { PropertyNameCaseInsensitive = true };
                                        var cvResponse = JsonSerializer.Deserialize<ComputerVisionResponse>(responseString, jsonOptions);
                                        if (cvResponse.Results.Count() > 0)
                                        {
                                            bmiValues.AddRange(cvResponse.Results.Select(a => a.BMI));
                                            ageValues.AddRange(cvResponse.Results.Select(a => a.Age));
                                        }
                                        //JsonDocument doc = JsonDocument.Parse(responseString);
                                        //var jsonResponse = doc.RootElement;
                                        //var bmi = jsonResponse.GetProperty("bmi").GetDecimal();
                                        //if (bmi > 0)
                                        //  bmiValues.Add(bmi);
                                        System.Threading.Thread.Sleep(1000);
                                    }
                                    catch(Exception ex)
                                    {
                                        Console.WriteLine($"Warn: {ex.ToString()}");
                                    }
                                }
                                if (bmiValues.Count == 0 || ageValues.Count == 0)
                                    continue;
                                var ageMetadata = new ImageMetadata()
                                {
                                    ForumPostID = post.ForumPostID,
                                    Name = "Age",
                                    ValueDecimal = ageValues.Average()
                                };
                                connection.Insert(ageMetadata, transaction);
                                var bmiMetadata = new ImageMetadata()
                                {
                                    ForumPostID = post.ForumPostID,
                                    Name = "BMI",
                                    ValueDecimal = bmiValues.Average()
                                };
                                connection.Insert(bmiMetadata, transaction);

                                transaction.Commit();
                                Console.WriteLine($"Commited age: {ageValues.Average()} and BMI: {bmiValues.Average()} for ForumPostID: {post.ForumPostID}");
                            }
                            catch (Exception ex)
                            {
                            }
                            
                        }
                    }
                }
            } while (posts.Count == 0 || posts.Min(posts => posts.Timestamp) > lastRun);
            System.IO.File.WriteAllText("lastrun", lastRun.ToString());
            Console.WriteLine("done");
        }
    }
}