chris/PVisionMetadataExtractor
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial project commit

Browse Source
This commit is contained in:
Chris
2021-02-15 11:38:37 -08:00
commit 0f1eb41b09
11 changed files with 692 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
PornocopiaVisionMetadataExtractor/Data/ComputerVisionResponse.cs Normal file
View File

@@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.Json.Serialization;
namespace PornocopiaVisionMetadataExtractor.Data
{
public class ComputerVisionResponse
{
[JsonPropertyName("request_id")]
public String RequestID { get; set; }
[JsonPropertyName("metadata")]
public ComputervisionFaceResult[] Results { get; set; }
}
}

17
PornocopiaVisionMetadataExtractor/Data/ComputervisionFaceResult.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.Json.Serialization;
namespace PornocopiaVisionMetadataExtractor.Data
{
public class ComputervisionFaceResult
{
public DetectedFace Face { get; set; }
public Decimal Age { get; set; }
public Decimal BMI { get; set; }
[JsonPropertyName("gender_woman")]
public Decimal GenderWomanPercentage { get; set; }
}
}

17
PornocopiaVisionMetadataExtractor/Data/DetectedFace.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace PornocopiaVisionMetadataExtractor.Data
{
public class DetectedFace
{
public Int32 Top { get; set; }
public Int32 Bottom { get; set; }
public Int32 Left { get; set; }
public Int32 Right { get; set; }
public Int32 Size { get; set; }
public Decimal SizePercentage { get; set; }
}
}

23
PornocopiaVisionMetadataExtractor/Data/ForumPost.cs Normal file
View File

@@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace PornocopiaVisionMetadataExtractor.Data
{
class ForumPost
{
public ForumPost()
{
Images = new List<ForumPostImage>();
}
public Int32 ForumPostID { get; set; }
public String Title { get; set; }
public String Link { get; set; }
public String Author { get; set; }
public DateTime Timestamp { get; set; }
public Boolean Downloadable { get; set; }
public Int32 Source { get; set; }
public Int32 PostID { get; set; }
public List<ForumPostImage> Images { get; set; }
}
}

17
PornocopiaVisionMetadataExtractor/Data/ForumPostImage.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace PornocopiaVisionMetadataExtractor.Data
{
class ForumPostImage
{
public Int32 ImageID { get; set; }
public Int32 ForumPostID { get; set; }
public Int32 TorrentPostID { get; set; }
public String ImageName { get; set; }
public String ImageLocation { get; set; }
public Int32 IndexerPostID { get; set; }
}
}

21
PornocopiaVisionMetadataExtractor/Data/ImageMetadata.cs Normal file
View File

@@ -0,0 +1,21 @@
using Dapper.Contrib.Extensions;
using System;
using System.Collections.Generic;
using System.Text;
namespace PornocopiaVisionMetadataExtractor.Data
{
[Table("porn_imagemetadata")]
class ImageMetadata
{
[Key]
public Int32 MetadataID { get; set; }
[ExplicitKey]
public Int32 ForumPostID { get; set; }
public String Name { get; set; }
public String ValueString { get; set; }
public Int32? ValueInt { get; set; }
public Decimal ValueDecimal { get; set; }
}
}

23
PornocopiaVisionMetadataExtractor/PornocopiaVisionMetadataExtractor.csproj Normal file
View File

@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.0.78" />
<PackageReference Include="Dapper.Contrib" Version="2.0.78" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
</ItemGroup>
<ItemGroup>
<None Update="appSettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

153
PornocopiaVisionMetadataExtractor/Program.cs Normal file
View File

@@ -0,0 +1,153 @@
using Microsoft.Extensions.Configuration;
using System;
using System.Data.SqlClient;
using Dapper;
using Dapper.Contrib;
using PornocopiaVisionMetadataExtractor.Data;
using System.Linq;
using System.Reflection;
using System.Collections.Generic;
using System.Transactions;
using Dapper.Contrib.Extensions;
using System.Net.Http;
using System.Text.Json;
namespace PornocopiaVisionMetadataExtractor
{
class Program
{
static void Main(string[] args)
{
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.SetBasePath(System.IO.Directory.GetCurrentDirectory()); // errors here
configurationBuilder.AddJsonFile(path: "appSettings.json", optional: false, reloadOnChange: true); // errors here
//configurationBuilder.AddXmlFile(Assembly.GetExecutingAssembly().Location + ".config", optional: false, reloadOnChange: true);
var config = configurationBuilder.Build();
DateTime lastRun = DateTime.Parse("2021-01-01");
try
{
String lastRuntime = System.IO.File.ReadAllText("lastrun");
DateTime.TryParse(lastRuntime, out lastRun);
}
catch (Exception) { }
List<ForumPost> posts;
DateTime? newestPost = null;
int page = 0;
int requestID = 1;
String connectionString = config.GetConnectionString("Pornocopia");
String baseImageServerURL = config.GetSection("AppSettings")["BaseImageServerURL"];
String VisionMetadataURL = config.GetSection("AppSettings")["VisionMetadataURL"];
do
{
using (SqlConnection connection = new SqlConnection(connectionString))
{
connection.Open();
var storedProcedureParameters = new DynamicParameters();
storedProcedureParameters.Add("pageSize", 100);
//temp dont increment because new posts will be filtered offsetting the page....
storedProcedureParameters.Add("currentPage", page++);
using (var multi = connection.QueryMultiple("GetMetadataMissingForumPosts", storedProcedureParameters, commandType: System.Data.CommandType.StoredProcedure))
{
posts = multi.Read<ForumPost>().ToList();
var images = multi.Read<ForumPostImage>();
foreach (var post in posts)
post.Images = images.Where(img => img.ForumPostID == post.ForumPostID).ToList();
}
}
if (!newestPost.HasValue)
newestPost = posts.Max(posts => posts.Timestamp);
var http = new System.Net.WebClient();
//Download image
using (SqlConnection connection = new SqlConnection(connectionString))
{
connection.Open();
foreach (var post in posts)
{
using (var transaction = connection.BeginTransaction())
{
List<Decimal> bmiValues = new List<decimal>();
List<Decimal> ageValues = new List<decimal>();
try
{
foreach (var image in post.Images)
{
try
{
String downloadURL = $"{baseImageServerURL}/{image.ImageLocation}";
byte[] data = http.DownloadData(downloadURL);
String submissionFilename = image.ImageLocation.Substring(image.ImageLocation.LastIndexOf('/') + 1);
//Do vision work
var httpVisionClient = new HttpClient();
MultipartFormDataContent form = new MultipartFormDataContent();
form.Add(new StringContent(requestID.ToString()), "request_id");
form.Add(new ByteArrayContent(data, 0, data.Length), "image_file", submissionFilename);
requestID++;
var httpTask = httpVisionClient.PostAsync(VisionMetadataURL, form);
httpTask.Wait();
var response = httpTask.Result;
var StringReadTask = response.Content.ReadAsStringAsync();
StringReadTask.Wait();
var responseString = StringReadTask.Result;
if (responseString.Contains("500 Internal Server Error"))
continue;
var jsonOptions = new JsonSerializerOptions() { PropertyNameCaseInsensitive = true };
var cvResponse = JsonSerializer.Deserialize<ComputerVisionResponse>(responseString, jsonOptions);
if (cvResponse.Results.Count() > 0)
{
bmiValues.AddRange(cvResponse.Results.Select(a => a.BMI));
ageValues.AddRange(cvResponse.Results.Select(a => a.Age));
}
//JsonDocument doc = JsonDocument.Parse(responseString);
//var jsonResponse = doc.RootElement;
//var bmi = jsonResponse.GetProperty("bmi").GetDecimal();
//if (bmi > 0)
// bmiValues.Add(bmi);
System.Threading.Thread.Sleep(1000);
}
catch(Exception ex)
{
Console.WriteLine($"Warn: {ex.ToString()}");
}
}
if (bmiValues.Count == 0 || ageValues.Count == 0)
continue;
var ageMetadata = new ImageMetadata()
{
ForumPostID = post.ForumPostID,
Name = "Age",
ValueDecimal = ageValues.Average()
};
connection.Insert(ageMetadata, transaction);
var bmiMetadata = new ImageMetadata()
{
ForumPostID = post.ForumPostID,
Name = "BMI",
ValueDecimal = bmiValues.Average()
};
connection.Insert(bmiMetadata, transaction);
transaction.Commit();
Console.WriteLine($"Commited age: {ageValues.Average()} and BMI: {bmiValues.Average()} for ForumPostID: {post.ForumPostID}");
}
catch (Exception ex)
{
}
}
}
}
} while (posts.Count == 0 || posts.Min(posts => posts.Timestamp) > lastRun);
System.IO.File.WriteAllText("lastrun", lastRun.ToString());
Console.WriteLine("done");
}
}
}

18
PornocopiaVisionMetadataExtractor/appSettings.json Normal file
View File

@@ -0,0 +1,18 @@
{
"exclude": [
"**/bin",
"**/bower_components",
"**/jspm_packages",
"**/node_modules",
"**/obj",
"**/platforms"
],
"ConnectionStrings": {
"Pornocopia": "Server=ORDO.chrispr.lan;Database=P;User Id=pornocopia;Password=aidsaids;"
},
"AppSettings": {
"BaseImageServerURL": "http://ordo.chrispr.lan/pics",
"VisionMetadataURL": "http://basilisk.chrispr.lan:4455/metadata/dev/calculate-image-metadata"
}
}