chris/PVisionMetadataExtractor
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial project commit

Browse Source
This commit is contained in:
Chris
2021-02-15 11:38:37 -08:00
commit 0f1eb41b09
11 changed files with 692 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

362
.gitignore vendored Normal file
View File

@@ -0,0 +1,362 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
##
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
# User-specific files
*.rsuser
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Mono auto generated files
mono_crash.*
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
[Ww][Ii][Nn]32/
[Aa][Rr][Mm]/
[Aa][Rr][Mm]64/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
[Ll]ogs/
# Visual Studio 2015/2017 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# Visual Studio 2017 auto generated files
Generated\ Files/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUnit
*.VisualState.xml
TestResult.xml
nunit-*.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# Benchmark Results
BenchmarkDotNet.Artifacts/
# .NET Core
project.lock.json
project.fragment.lock.json
artifacts/
# ASP.NET Scaffolding
ScaffoldingReadMe.txt
# StyleCop
StyleCopReport.xml
# Files built by Visual Studio
*_i.c
*_p.c
*_h.h
*.ilk
*.meta
*.obj
*.iobj
*.pch
*.pdb
*.ipdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*_wpftmp.csproj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# Visual Studio Trace Files
*.e2e
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# AxoCover is a Code Coverage Tool
.axoCover/*
!.axoCover/settings.json
# Coverlet is a free, cross platform Code Coverage Tool
coverage*.json
coverage*.xml
coverage*.info
# Visual Studio code coverage results
*.coverage
*.coveragexml
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# Note: Comment the next line if you want to checkin your web deploy settings,
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# NuGet Symbol Packages
*.snupkg
# The packages folder can be ignored because of Package Restore
**/[Pp]ackages/*
# except build/, which is used as an MSBuild target.
!**/[Pp]ackages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/[Pp]ackages/repositories.config
# NuGet v3's project.json files produces more ignorable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
*.appx
*.appxbundle
*.appxupload
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!?*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
orleans.codegen.cs
# Including strong name files can present a security risk
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
#*.snk
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
ServiceFabricBackup/
*.rptproj.bak
# SQL Server files
*.mdf
*.ldf
*.ndf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
*.rptproj.rsuser
*- [Bb]ackup.rdl
*- [Bb]ackup ([0-9]).rdl
*- [Bb]ackup ([0-9][0-9]).rdl
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
node_modules/
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# CodeRush personal settings
.cr/personal
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Cake - Uncomment if you are using it
# tools/**
# !tools/packages.config
# Tabs Studio
*.tss
# Telerik's JustMock configuration file
*.jmconfig
# BizTalk build output
*.btp.cs
*.btm.cs
*.odx.cs
*.xsd.cs
# OpenCover UI analysis results
OpenCover/
# Azure Stream Analytics local run output
ASALocalRun/
# MSBuild Binary and Structured Log
*.binlog
# NVidia Nsight GPU debugger configuration file
*.nvuser
# MFractors (Xamarin productivity tool) working folder
.mfractor/
# Local History for Visual Studio
.localhistory/
# BeatPulse healthcheck temp database
healthchecksdb
# Backup folder for Package Reference Convert tool in Visual Studio 2017
MigrationBackup/
# Ionide (cross platform F# VS Code tools) working folder
.ionide/
# Fody - auto-generated XML schema
FodyWeavers.xsd

25
PornocopiaVisionMetadataExtractor.sln Normal file
View File

@@ -0,0 +1,25 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.31005.135
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PornocopiaVisionMetadataExtractor", "PornocopiaVisionMetadataExtractor\PornocopiaVisionMetadataExtractor.csproj", "{053D2594-A33F-401D-B95C-441D931EEE47}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{053D2594-A33F-401D-B95C-441D931EEE47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{053D2594-A33F-401D-B95C-441D931EEE47}.Debug|Any CPU.Build.0 = Debug|Any CPU
{053D2594-A33F-401D-B95C-441D931EEE47}.Release|Any CPU.ActiveCfg = Release|Any CPU
{053D2594-A33F-401D-B95C-441D931EEE47}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {7B1A71CA-845E-453F-A69A-2640AA6FA908}
EndGlobalSection
EndGlobal

16
PornocopiaVisionMetadataExtractor/Data/ComputerVisionResponse.cs Normal file
View File

@@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.Json.Serialization;
namespace PornocopiaVisionMetadataExtractor.Data
{
public class ComputerVisionResponse
{
[JsonPropertyName("request_id")]
public String RequestID { get; set; }
[JsonPropertyName("metadata")]
public ComputervisionFaceResult[] Results { get; set; }
}
}

17
PornocopiaVisionMetadataExtractor/Data/ComputervisionFaceResult.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.Json.Serialization;
namespace PornocopiaVisionMetadataExtractor.Data
{
public class ComputervisionFaceResult
{
public DetectedFace Face { get; set; }
public Decimal Age { get; set; }
public Decimal BMI { get; set; }
[JsonPropertyName("gender_woman")]
public Decimal GenderWomanPercentage { get; set; }
}
}

17
PornocopiaVisionMetadataExtractor/Data/DetectedFace.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace PornocopiaVisionMetadataExtractor.Data
{
public class DetectedFace
{
public Int32 Top { get; set; }
public Int32 Bottom { get; set; }
public Int32 Left { get; set; }
public Int32 Right { get; set; }
public Int32 Size { get; set; }
public Decimal SizePercentage { get; set; }
}
}

23
PornocopiaVisionMetadataExtractor/Data/ForumPost.cs Normal file
View File

@@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace PornocopiaVisionMetadataExtractor.Data
{
class ForumPost
{
public ForumPost()
{
Images = new List<ForumPostImage>();
}
public Int32 ForumPostID { get; set; }
public String Title { get; set; }
public String Link { get; set; }
public String Author { get; set; }
public DateTime Timestamp { get; set; }
public Boolean Downloadable { get; set; }
public Int32 Source { get; set; }
public Int32 PostID { get; set; }
public List<ForumPostImage> Images { get; set; }
}
}

17
PornocopiaVisionMetadataExtractor/Data/ForumPostImage.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace PornocopiaVisionMetadataExtractor.Data
{
class ForumPostImage
{
public Int32 ImageID { get; set; }
public Int32 ForumPostID { get; set; }
public Int32 TorrentPostID { get; set; }
public String ImageName { get; set; }
public String ImageLocation { get; set; }
public Int32 IndexerPostID { get; set; }
}
}

21
PornocopiaVisionMetadataExtractor/Data/ImageMetadata.cs Normal file
View File

@@ -0,0 +1,21 @@
using Dapper.Contrib.Extensions;
using System;
using System.Collections.Generic;
using System.Text;
namespace PornocopiaVisionMetadataExtractor.Data
{
[Table("porn_imagemetadata")]
class ImageMetadata
{
[Key]
public Int32 MetadataID { get; set; }
[ExplicitKey]
public Int32 ForumPostID { get; set; }
public String Name { get; set; }
public String ValueString { get; set; }
public Int32? ValueInt { get; set; }
public Decimal ValueDecimal { get; set; }
}
}

23
PornocopiaVisionMetadataExtractor/PornocopiaVisionMetadataExtractor.csproj Normal file
View File

@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.0.78" />
<PackageReference Include="Dapper.Contrib" Version="2.0.78" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
</ItemGroup>
<ItemGroup>
<None Update="appSettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

153
PornocopiaVisionMetadataExtractor/Program.cs Normal file
View File

@@ -0,0 +1,153 @@
using Microsoft.Extensions.Configuration;
using System;
using System.Data.SqlClient;
using Dapper;
using Dapper.Contrib;
using PornocopiaVisionMetadataExtractor.Data;
using System.Linq;
using System.Reflection;
using System.Collections.Generic;
using System.Transactions;
using Dapper.Contrib.Extensions;
using System.Net.Http;
using System.Text.Json;
namespace PornocopiaVisionMetadataExtractor
{
class Program
{
static void Main(string[] args)
{
var configurationBuilder = new ConfigurationBuilder();
configurationBuilder.SetBasePath(System.IO.Directory.GetCurrentDirectory()); // errors here
configurationBuilder.AddJsonFile(path: "appSettings.json", optional: false, reloadOnChange: true); // errors here
//configurationBuilder.AddXmlFile(Assembly.GetExecutingAssembly().Location + ".config", optional: false, reloadOnChange: true);
var config = configurationBuilder.Build();
DateTime lastRun = DateTime.Parse("2021-01-01");
try
{
String lastRuntime = System.IO.File.ReadAllText("lastrun");
DateTime.TryParse(lastRuntime, out lastRun);
}
catch (Exception) { }
List<ForumPost> posts;
DateTime? newestPost = null;
int page = 0;
int requestID = 1;
String connectionString = config.GetConnectionString("Pornocopia");
String baseImageServerURL = config.GetSection("AppSettings")["BaseImageServerURL"];
String VisionMetadataURL = config.GetSection("AppSettings")["VisionMetadataURL"];
do
{
using (SqlConnection connection = new SqlConnection(connectionString))
{
connection.Open();
var storedProcedureParameters = new DynamicParameters();
storedProcedureParameters.Add("pageSize", 100);
//temp dont increment because new posts will be filtered offsetting the page....
storedProcedureParameters.Add("currentPage", page++);
using (var multi = connection.QueryMultiple("GetMetadataMissingForumPosts", storedProcedureParameters, commandType: System.Data.CommandType.StoredProcedure))
{
posts = multi.Read<ForumPost>().ToList();
var images = multi.Read<ForumPostImage>();
foreach (var post in posts)
post.Images = images.Where(img => img.ForumPostID == post.ForumPostID).ToList();
}
}
if (!newestPost.HasValue)
newestPost = posts.Max(posts => posts.Timestamp);
var http = new System.Net.WebClient();
//Download image
using (SqlConnection connection = new SqlConnection(connectionString))
{
connection.Open();
foreach (var post in posts)
{
using (var transaction = connection.BeginTransaction())
{
List<Decimal> bmiValues = new List<decimal>();
List<Decimal> ageValues = new List<decimal>();
try
{
foreach (var image in post.Images)
{
try
{
String downloadURL = $"{baseImageServerURL}/{image.ImageLocation}";
byte[] data = http.DownloadData(downloadURL);
String submissionFilename = image.ImageLocation.Substring(image.ImageLocation.LastIndexOf('/') + 1);
//Do vision work
var httpVisionClient = new HttpClient();
MultipartFormDataContent form = new MultipartFormDataContent();
form.Add(new StringContent(requestID.ToString()), "request_id");
form.Add(new ByteArrayContent(data, 0, data.Length), "image_file", submissionFilename);
requestID++;
var httpTask = httpVisionClient.PostAsync(VisionMetadataURL, form);
httpTask.Wait();
var response = httpTask.Result;
var StringReadTask = response.Content.ReadAsStringAsync();
StringReadTask.Wait();
var responseString = StringReadTask.Result;
if (responseString.Contains("500 Internal Server Error"))
continue;
var jsonOptions = new JsonSerializerOptions() { PropertyNameCaseInsensitive = true };
var cvResponse = JsonSerializer.Deserialize<ComputerVisionResponse>(responseString, jsonOptions);
if (cvResponse.Results.Count() > 0)
{
bmiValues.AddRange(cvResponse.Results.Select(a => a.BMI));
ageValues.AddRange(cvResponse.Results.Select(a => a.Age));
}
//JsonDocument doc = JsonDocument.Parse(responseString);
//var jsonResponse = doc.RootElement;
//var bmi = jsonResponse.GetProperty("bmi").GetDecimal();
//if (bmi > 0)
// bmiValues.Add(bmi);
System.Threading.Thread.Sleep(1000);
}
catch(Exception ex)
{
Console.WriteLine($"Warn: {ex.ToString()}");
}
}
if (bmiValues.Count == 0 || ageValues.Count == 0)
continue;
var ageMetadata = new ImageMetadata()
{
ForumPostID = post.ForumPostID,
Name = "Age",
ValueDecimal = ageValues.Average()
};
connection.Insert(ageMetadata, transaction);
var bmiMetadata = new ImageMetadata()
{
ForumPostID = post.ForumPostID,
Name = "BMI",
ValueDecimal = bmiValues.Average()
};
connection.Insert(bmiMetadata, transaction);
transaction.Commit();
Console.WriteLine($"Commited age: {ageValues.Average()} and BMI: {bmiValues.Average()} for ForumPostID: {post.ForumPostID}");
}
catch (Exception ex)
{
}
}
}
}
} while (posts.Count == 0 || posts.Min(posts => posts.Timestamp) > lastRun);
System.IO.File.WriteAllText("lastrun", lastRun.ToString());
Console.WriteLine("done");
}
}
}

18
PornocopiaVisionMetadataExtractor/appSettings.json Normal file
View File

@@ -0,0 +1,18 @@
{
"exclude": [
"**/bin",
"**/bower_components",
"**/jspm_packages",
"**/node_modules",
"**/obj",
"**/platforms"
],
"ConnectionStrings": {
"Pornocopia": "Server=ORDO.chrispr.lan;Database=P;User Id=pornocopia;Password=aidsaids;"
},
"AppSettings": {
"BaseImageServerURL": "http://ordo.chrispr.lan/pics",
"VisionMetadataURL": "http://basilisk.chrispr.lan:4455/metadata/dev/calculate-image-metadata"
}
}