| | | 1 | | using RoleRadar.Application.Abstractions; |
| | | 2 | | using RoleRadar.Domain.Tracking; |
| | | 3 | | |
| | | 4 | | namespace RoleRadar.Application.Tracking.Import.Pipeline; |
| | | 5 | | |
| | | 6 | | public sealed class TrackerImportContext |
| | | 7 | | { |
| | | 8 | | public required string OwnerId { get; init; } |
| | | 9 | | public required string BundleDirectory { get; init; } |
| | | 10 | | public DateOnly? DefaultDate { get; init; } |
| | | 11 | | public bool Apply { get; init; } |
| | | 12 | | public TrackerImportSummary Summary { get; } = new(); |
| | | 13 | | |
| | | 14 | | public Dictionary<string, string> CompanyAliases { get; } = new(StringComparer.OrdinalIgnoreCase); |
| | | 15 | | public Dictionary<string, Company> CanonicalCompanies { get; } = new(StringComparer.OrdinalIgnoreCase); |
| | | 16 | | public Dictionary<string, Posting> PostingsByUrl { get; } = new(StringComparer.OrdinalIgnoreCase); |
| | | 17 | | public Dictionary<string, Posting> PostingsByCompanyAndTitle { get; } = new(StringComparer.OrdinalIgnoreCase); |
| | | 18 | | public Dictionary<string, Posting> PostingsByCompanyAndReq { get; } = new(StringComparer.OrdinalIgnoreCase); |
| | | 19 | | public Dictionary<Guid, Posting> AllPostings { get; } = new(); |
| | | 20 | | public HashSet<Guid> ClaimedPostingIds { get; } = new(); |
| | | 21 | | public HashSet<Guid> ClaimedPostingIdsInRun { get; } = new(); |
| | | 22 | | |
| | | 23 | | public void IndexPosting(Posting posting, string? canonicalCompanyName) |
| | | 24 | | { |
| | | 25 | | AllPostings[posting.Id] = posting; |
| | | 26 | | if (!string.IsNullOrWhiteSpace(posting.OriginalUrl)) |
| | | 27 | | { |
| | | 28 | | var normUrl = UrlNormalizer.Normalize(posting.OriginalUrl); |
| | | 29 | | if (normUrl is not null) |
| | | 30 | | { |
| | | 31 | | PostingsByUrl[normUrl] = posting; |
| | | 32 | | } |
| | | 33 | | PostingsByUrl[posting.OriginalUrl] = posting; |
| | | 34 | | } |
| | | 35 | | |
| | | 36 | | if (!string.IsNullOrWhiteSpace(posting.RequisitionId)) |
| | | 37 | | { |
| | | 38 | | PostingsByCompanyAndReq[$"{posting.CompanyId:D}|{posting.RequisitionId.Trim()}"] = posting; |
| | | 39 | | if (!string.IsNullOrWhiteSpace(canonicalCompanyName)) |
| | | 40 | | { |
| | | 41 | | PostingsByCompanyAndReq[$"{canonicalCompanyName}|{posting.RequisitionId.Trim()}"] = posting; |
| | | 42 | | } |
| | | 43 | | } |
| | | 44 | | |
| | | 45 | | if (!string.IsNullOrWhiteSpace(canonicalCompanyName) && !string.IsNullOrWhiteSpace(posting.NormalizedTitle)) |
| | | 46 | | { |
| | | 47 | | PostingsByCompanyAndTitle[$"{canonicalCompanyName}|{posting.NormalizedTitle}"] = posting; |
| | | 48 | | } |
| | | 49 | | } |
| | | 50 | | } |
| | | 51 | | |
| | | 52 | | public interface ITrackerImportStep |
| | | 53 | | { |
| | | 54 | | Task<Result<bool>> ExecuteAsync(TrackerImportContext context, CancellationToken cancellationToken = default); |
| | | 55 | | } |
| | | 56 | | |
| | | 57 | | public interface ITrackerImportPipelineFactory |
| | | 58 | | { |
| | | 59 | | TrackerImportPipeline CreatePipeline(ITrackingTransactionContext context); |
| | | 60 | | } |
| | | 61 | | |
| | | 62 | | public sealed class TrackerImportPipelineFactory : ITrackerImportPipelineFactory |
| | | 63 | | { |
| | | 64 | | private readonly ICsvParser _csvParser; |
| | | 65 | | private readonly Func<int, Task>? _applicationsFaultHook; |
| | | 66 | | |
| | | 67 | | public TrackerImportPipelineFactory(ICsvParser csvParser, Func<int, Task>? applicationsFaultHook = null) |
| | | 68 | | { |
| | | 69 | | _csvParser = csvParser ?? throw new ArgumentNullException(nameof(csvParser)); |
| | | 70 | | _applicationsFaultHook = applicationsFaultHook; |
| | | 71 | | } |
| | | 72 | | |
| | | 73 | | public TrackerImportPipeline CreatePipeline(ITrackingTransactionContext context) |
| | | 74 | | { |
| | | 75 | | ArgumentNullException.ThrowIfNull(context); |
| | | 76 | | |
| | | 77 | | var criteriaStep = new CriteriaSeedingStep(context); |
| | | 78 | | var aliasesStep = new CompanyAliasesStep(_csvParser); |
| | | 79 | | var companiesStep = new WatchlistCompaniesStep(_csvParser, context); |
| | | 80 | | var postingsStep = new PostingsJsonStep(context, criteriaStep); |
| | | 81 | | var appendixStep = new AppendixRowsStep(_csvParser, context, criteriaStep); |
| | | 82 | | var applicationsStep = new ApplicationsStep(_csvParser, context, _applicationsFaultHook); |
| | | 83 | | |
| | | 84 | | var steps = new ITrackerImportStep[] |
| | | 85 | | { |
| | | 86 | | new CriteriaSeedingPipelineStep(criteriaStep), |
| | | 87 | | aliasesStep, |
| | | 88 | | companiesStep, |
| | | 89 | | postingsStep, |
| | | 90 | | appendixStep, |
| | | 91 | | applicationsStep |
| | | 92 | | }; |
| | | 93 | | |
| | | 94 | | return new TrackerImportPipeline(steps, context); |
| | | 95 | | } |
| | | 96 | | } |
| | | 97 | | |
| | | 98 | | internal sealed class CriteriaSeedingPipelineStep : ITrackerImportStep |
| | | 99 | | { |
| | | 100 | | private readonly CriteriaSeedingStep _step; |
| | | 101 | | |
| | 40 | 102 | | public CriteriaSeedingPipelineStep(CriteriaSeedingStep step) |
| | | 103 | | { |
| | 40 | 104 | | _step = step; |
| | 40 | 105 | | } |
| | | 106 | | |
| | | 107 | | public async Task<Result<bool>> ExecuteAsync(TrackerImportContext context, CancellationToken cancellationToken = def |
| | | 108 | | { |
| | | 109 | | // When --default-date is absent, use today's UTC date as the effective date for seeded criteria. |
| | 40 | 110 | | var effectiveDate = context.DefaultDate ?? DateOnly.FromDateTime(DateTime.UtcNow); |
| | | 111 | | |
| | 40 | 112 | | var seedResult = await _step.SeedPredefinedCriteriaAsync(context.OwnerId, effectiveDate, cancellationToken); |
| | 40 | 113 | | if (!seedResult.IsSuccess) |
| | | 114 | | { |
| | 0 | 115 | | return Result<bool>.Failure(seedResult.Error!.Value); |
| | | 116 | | } |
| | | 117 | | |
| | 40 | 118 | | if (seedResult.Value is not null) |
| | | 119 | | { |
| | 40 | 120 | | context.Summary.CriteriaCreated += seedResult.Value.CreatedCount; |
| | | 121 | | } |
| | | 122 | | |
| | 40 | 123 | | return Result<bool>.Success(true); |
| | 40 | 124 | | } |
| | | 125 | | } |
| | | 126 | | |
| | | 127 | | public sealed class TrackerImportPipeline |
| | | 128 | | { |
| | | 129 | | private readonly IReadOnlyList<ITrackerImportStep> _steps; |
| | | 130 | | private readonly ITrackingTransactionContext? _trackingContext; |
| | | 131 | | |
| | | 132 | | public TrackerImportPipeline(IReadOnlyList<ITrackerImportStep> steps, ITrackingTransactionContext? trackingContext = |
| | | 133 | | { |
| | | 134 | | _steps = steps ?? throw new ArgumentNullException(nameof(steps)); |
| | | 135 | | _trackingContext = trackingContext; |
| | | 136 | | } |
| | | 137 | | |
| | | 138 | | public async Task<Result<TrackerImportSummary>> ExecuteAsync( |
| | | 139 | | TrackerImportContext context, |
| | | 140 | | CancellationToken cancellationToken = default) |
| | | 141 | | { |
| | | 142 | | ArgumentNullException.ThrowIfNull(context); |
| | | 143 | | |
| | | 144 | | var knownFiles = new[] |
| | | 145 | | { |
| | | 146 | | "company-aliases.csv", |
| | | 147 | | "watchlist-companies.csv", |
| | | 148 | | "postings.json", |
| | | 149 | | "appendix-rows.csv", |
| | | 150 | | "applications.csv" |
| | | 151 | | }; |
| | | 152 | | |
| | | 153 | | foreach (var file in knownFiles) |
| | | 154 | | { |
| | | 155 | | var fullPath = Path.Combine(context.BundleDirectory, file); |
| | | 156 | | var recon = context.Summary.GetOrCreateFileReconciliation(file); |
| | | 157 | | if (!File.Exists(fullPath)) |
| | | 158 | | { |
| | | 159 | | recon.IsAbsent = true; |
| | | 160 | | } |
| | | 161 | | } |
| | | 162 | | |
| | | 163 | | foreach (var step in _steps) |
| | | 164 | | { |
| | | 165 | | var result = await step.ExecuteAsync(context, cancellationToken); |
| | | 166 | | if (!result.IsSuccess) |
| | | 167 | | { |
| | | 168 | | return Result<TrackerImportSummary>.Failure(result.Error!.Value); |
| | | 169 | | } |
| | | 170 | | } |
| | | 171 | | |
| | | 172 | | context.Summary.DispositionHistogram.Clear(); |
| | | 173 | | |
| | | 174 | | if (_trackingContext is not null) |
| | | 175 | | { |
| | | 176 | | var page = 1; |
| | | 177 | | while (true) |
| | | 178 | | { |
| | | 179 | | var existingRes = await _trackingContext.Postings.ListPostingsAsync(context.OwnerId, page: page, pageSiz |
| | | 180 | | if (!existingRes.IsSuccess || existingRes.Value is null || existingRes.Value.Count == 0) |
| | | 181 | | { |
| | | 182 | | break; |
| | | 183 | | } |
| | | 184 | | foreach (var p in existingRes.Value) |
| | | 185 | | { |
| | | 186 | | if (!context.AllPostings.ContainsKey(p.Id)) |
| | | 187 | | { |
| | | 188 | | context.AllPostings[p.Id] = p; |
| | | 189 | | } |
| | | 190 | | } |
| | | 191 | | if (existingRes.Value.Count < 200) |
| | | 192 | | { |
| | | 193 | | break; |
| | | 194 | | } |
| | | 195 | | page++; |
| | | 196 | | } |
| | | 197 | | } |
| | | 198 | | |
| | | 199 | | foreach (var p in context.AllPostings.Values) |
| | | 200 | | { |
| | | 201 | | var dispName = p.Disposition.ToSnakeCase(); |
| | | 202 | | context.Summary.DispositionHistogram.TryGetValue(dispName, out var count); |
| | | 203 | | context.Summary.DispositionHistogram[dispName] = count + 1; |
| | | 204 | | } |
| | | 205 | | |
| | | 206 | | return Result<TrackerImportSummary>.Success(context.Summary); |
| | | 207 | | } |
| | | 208 | | } |