preparing for fen db

This commit is contained in:
Adrian Stephens 2026-05-09 07:28:18 +01:00
parent 06fddc641f
commit e02696ea2f
9 changed files with 538 additions and 189 deletions

View File

@ -3,7 +3,7 @@
* Plugin Name: FECA Mailshots
* Plugin URI: https://fenedge.co.uk/
* Description: FECA mailshots plugin.
* Version: 1.0.0
* Version: 1.0.3
* Requires at least: 6.0
* Requires PHP: 7.4
* Author: FECA

View File

@ -145,12 +145,8 @@ final class DataSourceService
$stmtRows = $this->router->membersPdo()->prepare($previewSql);
$stmtRows->execute($compiled['params']);
$rows = $stmtRows->fetchAll(PDO::FETCH_ASSOC);
$rows = $this->withExpectedFieldAliases($rows, $validation['expected_fields']);
$columnSet = [];
foreach ($validation['expected_fields'] as $field) {
$columnSet[$field] = true;
}
foreach ($rows as $row) {
foreach (array_keys($row) as $key) {
$columnSet[$key] = true;
@ -167,74 +163,6 @@ final class DataSourceService
];
}
/**
* Ensure preview rows expose the same qualified field names used by token generation.
*
* @param list<array<string,mixed>> $rows
* @param list<string> $expectedFields
* @return list<array<string,mixed>>
*/
private function withExpectedFieldAliases(array $rows, array $expectedFields): array
{
if ($rows === [] || $expectedFields === []) {
return $rows;
}
$expectedByLeaf = [];
foreach ($expectedFields as $qualified) {
$qualified = trim((string) $qualified);
if ($qualified === '') {
continue;
}
$leaf = $this->leafFieldName($qualified);
if ($leaf === '') {
continue;
}
$expectedByLeaf[strtolower($leaf)][] = $qualified;
}
if ($expectedByLeaf === []) {
return $rows;
}
foreach ($rows as &$row) {
if (!is_array($row) || $row === []) {
continue;
}
$keyLookup = [];
foreach (array_keys($row) as $key) {
$k = (string) $key;
$keyLookup[strtolower($k)] = $k;
}
foreach ($expectedByLeaf as $leafLower => $qualifiedList) {
if (!isset($keyLookup[$leafLower])) {
continue;
}
$sourceKey = $keyLookup[$leafLower];
foreach ($qualifiedList as $qualified) {
if (array_key_exists($qualified, $row)) {
continue;
}
$row[$qualified] = $row[$sourceKey];
}
}
}
unset($row);
return $rows;
}
private function leafFieldName(string $qualifiedField): string
{
$pos = strrpos($qualifiedField, '.');
if ($pos === false) {
return $qualifiedField;
}
return substr($qualifiedField, $pos + 1);
}
/** @return array<string, list<string>> */
public function sourceFields(): array
{

View File

@ -57,7 +57,9 @@ final class DslCompiler
$whereSql = $whereParts === [] ? '' : ' WHERE ' . implode(' AND ', $whereParts);
$selectSql = count($sources) > 1 ? $this->buildUniqueSelectProjection($sources) : '*';
// Always emit source-qualified projection keys so token names remain stable
// without any post-query alias fallback.
$selectSql = $this->buildUniqueSelectProjection($sources);
$sql = 'SELECT ' . $selectSql . ' FROM ' . $from;
if ($joins !== []) {
$sql .= ' ' . implode(' ', $joins);

View File

@ -265,6 +265,11 @@ final class MailshotRunService
/** @return array<string, mixed> */
public function runMailshot(int $mailshotId): array
{
$mailshot = $this->mailshots->find($mailshotId);
if ($mailshot === null) {
return ['ok' => false, 'errors' => ['Mailshot not found.']];
}
$creds = $this->credentials->credentials();
if ($creds === null) {
return ['ok' => false, 'errors' => ['Missing mail credentials. Configure FECA Mailshots Profile page first.']];
@ -396,10 +401,14 @@ final class MailshotRunService
}
$files = [];
$mergedHtmlTmpPath = null;
$mergedHtmlHandle = null;
$ghostscriptBinary = $this->findExecutableBinary('gs', ['/usr/bin/gs', '/bin/gs']);
$useGhostscriptMerge = $includeMerged && $ghostscriptBinary !== '' && function_exists('exec');
$ghostscriptBinary = $this->findExecutableBinary('gs');
if ($includeMerged && $ghostscriptBinary === '') {
return ['ok' => false, 'errors' => ['Merged PDF generation requires Ghostscript (gs) to be installed and available on PATH.']];
}
if ($includeMerged && !function_exists('exec')) {
return ['ok' => false, 'errors' => ['Merged PDF generation requires PHP exec() to be enabled.']];
}
$useGhostscriptMerge = $includeMerged;
$mergedPdfTmpDir = '';
$mergedPdfPartPaths = [];
$nameCounts = [];
@ -412,28 +421,6 @@ final class MailshotRunService
if (!@mkdir($mergedPdfTmpDir, 0700, true) && !is_dir($mergedPdfTmpDir)) {
return ['ok' => false, 'errors' => ['Unable to prepare temporary directory for merged PDF build.']];
}
} elseif ($includeMerged) {
$tmpBasePath = tempnam(sys_get_temp_dir(), 'feca_mailshots_merged_html_');
if (!is_string($tmpBasePath) || $tmpBasePath === '') {
return ['ok' => false, 'errors' => ['Unable to allocate temporary merged HTML file.']];
}
$mergedHtmlTmpPath = $tmpBasePath . '.html';
if (!@rename($tmpBasePath, $mergedHtmlTmpPath)) {
@unlink($tmpBasePath);
return ['ok' => false, 'errors' => ['Unable to prepare temporary merged HTML file.']];
}
$mergedHtmlHandle = @fopen($mergedHtmlTmpPath, 'wb');
if (!is_resource($mergedHtmlHandle)) {
@unlink($mergedHtmlTmpPath);
return ['ok' => false, 'errors' => ['Unable to open temporary merged HTML file for writing.']];
}
fwrite(
$mergedHtmlHandle,
'<!doctype html><html><head><meta charset="UTF-8"><style>'
. '.feca-mailshot-pdf-section{page-break-after:always;}'
. '.feca-mailshot-pdf-section:last-child{page-break-after:auto;}'
. '</style></head><body>'
);
}
foreach (array_values($rows) as $index => $row) {
@ -468,9 +455,6 @@ final class MailshotRunService
$mergedPdfPartPaths[] = $partPath;
$mergedSectionCount++;
unset($pdfBytesForMerge);
} elseif ($includeMerged) {
fwrite($mergedHtmlHandle, '<div class="feca-mailshot-pdf-section">' . $pdfHtml . '</div>');
$mergedSectionCount++;
}
unset($rendered, $pdfHtml);
} catch (\Throwable $e) {
@ -482,37 +466,21 @@ final class MailshotRunService
}
}
if ($includeMerged && is_resource($mergedHtmlHandle)) {
fwrite($mergedHtmlHandle, '</body></html>');
fclose($mergedHtmlHandle);
$mergedHtmlHandle = null;
}
if ($errors !== []) {
if ($mergedHtmlTmpPath !== null) {
@unlink($mergedHtmlTmpPath);
}
$this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths);
return ['ok' => false, 'errors' => $errors];
}
if ($includeFiles && $files === []) {
if ($mergedHtmlTmpPath !== null) {
@unlink($mergedHtmlTmpPath);
}
$this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths);
return ['ok' => false, 'errors' => ['No PDF attachments were generated from this mailshot.']];
}
if ($includeMerged && $mergedSectionCount === 0) {
if ($mergedHtmlTmpPath !== null) {
@unlink($mergedHtmlTmpPath);
}
$this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths);
return ['ok' => false, 'errors' => ['No merged PDF content was generated from this mailshot.']];
}
$mergedPdfBytes = '';
if ($includeMerged) {
if ($useGhostscriptMerge) {
try {
$mergedPdfBytes = $this->mergePdfFilesWithGhostscript($ghostscriptBinary, $mergedPdfPartPaths);
} catch (\Throwable $e) {
@ -520,19 +488,6 @@ final class MailshotRunService
return ['ok' => false, 'errors' => ['Merged PDF generation failed: ' . $e->getMessage()]];
}
$this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths);
} else {
try {
$mergedPdfBytes = $this->renderPdfBytesFromHtmlFile((string) $mergedHtmlTmpPath);
} catch (\Throwable $e) {
if ($mergedHtmlTmpPath !== null) {
@unlink($mergedHtmlTmpPath);
}
return ['ok' => false, 'errors' => ['Merged PDF generation failed: ' . $e->getMessage()]];
}
if ($mergedHtmlTmpPath !== null) {
@unlink($mergedHtmlTmpPath);
}
}
}
return [
@ -549,7 +504,7 @@ final class MailshotRunService
public function generatePdfZipToTemp(int $mailshotId): array
{
$canUseZipArchive = class_exists('ZipArchive');
$zipBinary = $this->findExecutableBinary('zip', ['/usr/bin/zip', '/bin/zip']);
$zipBinary = $this->findExecutableBinary('zip');
if (!$canUseZipArchive && $zipBinary === '') {
return ['ok' => false, 'errors' => ['Neither ZipArchive extension nor zip CLI binary is available.']];
}
@ -707,7 +662,7 @@ final class MailshotRunService
}
}
private function findExecutableBinary(string $binaryName, array $fallbackPaths = []): string
private function findExecutableBinary(string $binaryName): string
{
$pathEnv = (string) getenv('PATH');
$candidates = [];
@ -719,9 +674,6 @@ final class MailshotRunService
}
}
}
foreach ($fallbackPaths as $p) {
$candidates[] = $p;
}
foreach ($candidates as $candidate) {
if (is_string($candidate) && $candidate !== '' && is_file($candidate) && is_executable($candidate)) {
@ -932,8 +884,9 @@ final class MailshotRunService
}
}
}
return null;
}
foreach (['Email', 'email', 'contact_email', 'account_email', 'recipient_email', 'recipient_email_last'] as $key) {
foreach (['Email', 'email', 'contact_email', 'recipient_email', 'recipient_email_last'] as $key) {
if (isset($row[$key])) {
$v = trim((string) $row[$key]);
if ($v !== '' && filter_var($v, FILTER_VALIDATE_EMAIL)) {

View File

@ -143,10 +143,10 @@ final class TemplateRenderer
*/
private function transcodePngToJpeg(string $assetName, string $pngBytes): array
{
$convertBinary = $this->findExecutableBinary('convert', ['/usr/bin/convert', '/bin/convert']);
$convertBinary = $this->findExecutableBinary('convert');
if ($convertBinary === '') {
throw new \RuntimeException(
'pdf_asset("' . $assetName . '") uses PNG, but this runtime lacks GD and ImageMagick convert.'
'pdf_asset("' . $assetName . '") uses PNG, but neither GD PNG support nor ImageMagick convert is available.'
);
}
if (!function_exists('exec')) {
@ -199,7 +199,7 @@ final class TemplateRenderer
}
}
private function findExecutableBinary(string $binaryName, array $fallbackPaths = []): string
private function findExecutableBinary(string $binaryName): string
{
$pathEnv = (string) getenv('PATH');
$candidates = [];
@ -211,9 +211,6 @@ final class TemplateRenderer
}
}
}
foreach ($fallbackPaths as $p) {
$candidates[] = $p;
}
foreach ($candidates as $candidate) {
if (is_string($candidate) && $candidate !== '' && is_file($candidate) && is_executable($candidate)) {

385
formats/renewals_v5.html Normal file
View File

@ -0,0 +1,385 @@
<!-- Template Version: 5 -->
<style>
* { box-sizing: border-box; }
@page { size: A4; margin: 10mm; }
body {
font-family: Arial, Helvetica, sans-serif;
color: #1f2937;
font-size: 9pt;
line-height: 1.25;
margin: 0;
padding: 0;
}
.page { width: 100%; position: relative; z-index: 1; }
.watermark {
position: fixed;
top: 50%;
left: 50%;
transform: translate(-50%, -50%) rotate(-35deg);
font-size: 72pt;
color: rgba(220, 38, 38, 0.12);
font-weight: bold;
letter-spacing: 7px;
z-index: 0;
pointer-events: none;
user-select: none;
white-space: nowrap;
}
.header {
margin-bottom: 5.5mm;
min-height: 29mm;
}
.logo-right {
float: right;
width: 42mm;
margin-left: 4mm;
text-align: right;
}
.header-text { overflow: hidden; }
.title {
font-size: 14.5pt;
font-weight: bold;
color: #0f172a;
margin: 0 0 1mm 0;
}
.subtitle {
margin: 0;
font-size: 9.2pt;
color: #4b5563;
}
.rule {
margin: 2.8mm 0 2.4mm 0;
border: 0;
border-top: 2px solid #d97706;
}
.fee-banner {
margin: 0 0 2.4mm 0;
padding: 1.9mm 2.5mm;
background: #fff7ed;
border-left: 4px solid #d97706;
font-size: 10pt;
font-weight: bold;
color: #9a3412;
}
.note {
margin: 0 0 1.3mm 0;
font-style: italic;
color: #374151;
}
.note-line {
margin: 0 0 3.8mm 0;
color: #111827;
font-weight: bold;
}
.pane {
border: 1px solid #d1d5db;
margin: 0 0 3.6mm 0;
}
.pane-title {
background: #f3f4f6;
border-bottom: 1px solid #d1d5db;
padding: 1.5mm 2.2mm;
font-weight: bold;
text-transform: uppercase;
letter-spacing: 0.4px;
font-size: 8pt;
color: #374151;
}
.pane-body { padding: 2.1mm 2.2mm; }
.two-col {
width: 100%;
border-collapse: collapse;
table-layout: fixed;
}
.two-col td {
width: 50%;
vertical-align: top;
padding: 0 1.5mm;
}
.two-col td:first-child { padding-left: 0; }
.two-col td:last-child { padding-right: 0; }
table.details {
width: 100%;
border-collapse: collapse;
table-layout: fixed;
}
table.details td {
border: 1px solid #e5e7eb;
padding: 1.5mm 2mm;
vertical-align: top;
white-space: normal;
overflow-wrap: anywhere;
word-break: break-word;
}
table.details td.label {
width: 40%;
font-weight: bold;
color: #374151;
background: #f9fafb;
}
table.details td.value {
width: 60%;
color: #111827;
font-weight: bold;
}
.instruction-box {
margin: 0 0 3.6mm 0;
border: 1px solid #d1d5db;
background: #f9fafb;
padding: 2mm 2.4mm;
color: #374151;
}
.sign {
width: 100%;
border-collapse: collapse;
margin-top: 3.2mm;
}
.sign td {
width: 50%;
border: 1px solid #d1d5db;
padding: 2.4mm 2.6mm;
vertical-align: bottom;
}
.sig-label {
font-size: 8.1pt;
font-weight: bold;
color: #374151;
margin-bottom: 4.8mm;
}
.sig-line {
border-bottom: 1px solid #111827;
height: 4.2mm;
}
.payments {
margin-top: 3.8mm;
border-top: 1px solid #d1d5db;
padding-top: 2.8mm;
}
.payments-title {
font-size: 9.2pt;
font-weight: bold;
margin: 0 0 2mm 0;
color: #111827;
}
.payments p { margin: 0 0 1.9mm 0; }
.clear { clear: both; }
</style>
<div class="watermark">DRAFT</div>
<div class="page">
<div class="header">
<div class="logo-right">{{ pdf_asset("Fen Edge Logo") }}</div>
<div class="header-text">
<p class="title">Membership Renewal 2026-2027</p>
<p class="subtitle">Fen Edge Community Association</p>
<hr class="rule">
<div class="fee-banner">Group Membership Fee: GBP15</div>
<p class="note">Please amend any details that have changed before returning this form.</p>
</div>
<div class="clear"><br></div>
</div>
<div class="pane">
<div class="pane-title">Account And Public Contact</div>
<div class="pane-body">
<table class="two-col">
<tbody>
<tr>
<td>
<table class="details">
<tbody>
<tr>
<td class="label">Account</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_account_name }}</td>
</tr>
<tr>
<td class="label">Type</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_account_type }}</td>
</tr>
<tr>
<td class="label">Sector</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_account_sector }}</td>
</tr>
<tr>
<td class="label">Public Location</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_public_location }}</td>
</tr>
<tr>
<td class="label">Members</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_account_no_of_members }}</td>
</tr>
</tbody>
</table>
</td>
<td>
<table class="details">
<tbody>
<tr>
<td class="label">Public Contact</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_public_contact_name }}</td>
</tr>
<tr>
<td class="label">Public Email</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_public_contact_public_email }}</td>
</tr>
<tr>
<td class="label">Public Phone</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_public_contact_public_phone }}</td>
</tr>
<tr>
<td class="label">Facebook</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_account_facebook }}</td>
</tr>
<tr>
<td class="label">Twitter</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_account_twitter }}</td>
</tr>
<tr>
<td class="label">Other Social</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_account_other_social }}</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="pane">
<div class="pane-title">Primary Contact (Contact 1)</div>
<div class="pane-body">
<table class="details">
<tbody>
<tr>
<td class="label">Name</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_name }}</td>
</tr>
<tr>
<td class="label">Position</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_position }}</td>
</tr>
<tr>
<td class="label">Address</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_address_1 }} {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_address_2 }}, {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_town }}, {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_county }}, {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_postcode }}</td>
</tr>
<tr>
<td class="label">Email</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_email }}</td>
</tr>
<tr>
<td class="label">Phone 1 / Phone 2</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_phone_1 }} {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_phone_2 }}</td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="pane">
<div class="pane-title">Secondary Contacts</div>
<div class="pane-body">
<table class="two-col">
<tbody>
<tr>
<td>
<table class="details">
<tbody>
<tr>
<td class="label">Contact 2 Name</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_contact_2_name }}</td>
</tr>
<tr>
<td class="label">Position</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_contact_2_position }}</td>
</tr>
<tr>
<td class="label">Email</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_contact_2_email }}</td>
</tr>
<tr>
<td class="label">Phone 1 / Phone 2</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_contact_2_phone_1 }} {{ fenedgec_members_renewal_accounts_with_contacts_contact_2_phone_2 }}</td>
</tr>
</tbody>
</table>
</td>
<td>
<table class="details">
<tbody>
<tr>
<td class="label">Fen Contact Name</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_fen_contact_name }}</td>
</tr>
<tr>
<td class="label">Fen Contact Email</td>
<td class="value">{{ fenedgec_members_renewal_accounts_with_contacts_fen_contact_email }}</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="instruction-box">
Please check your groups public contact details on the FECA website:
<strong>www.fenedge.co.uk/full-members-list</strong> and advise any changes.
</div>
<table class="sign">
<tbody>
<tr>
<td>
<div class="sig-label">Signed</div>
<div class="sig-line"><br></div>
</td>
<td>
<div class="sig-label">Date</div>
<div class="sig-line"><br></div>
</td>
</tr>
</tbody>
</table>
<div class="payments">
<p class="payments-title">Payment Options</p>
<p><strong>1. Cheque:</strong> payable to <strong>Fen Edge Community Association</strong>, sent with this form to <strong>FECA Community Office, Cottenham Village College, Cottenham, CB24 8UA</strong>.</p>
<p><strong>2. BACS:</strong> <strong>Sort Code 20-17-22</strong>, <strong>Account No. 40349178</strong>; then scan and email the form to <strong>info@fenedge.co.uk</strong>.</p>
</div>
</div>

View File

@ -7,11 +7,13 @@ Document requirements for a safe, non-admin-friendly mailshot data source defini
## Environment baseline for this specification
`members` as a database name means the value of `MEMBERS_REMOTE_MYSQL_DB`.
`fen` as a database name means the value of `FEN_REMOTE_MYSQL_DB`.
For this specification:
* Data-source definition persistence (create/update/delete/list of `mailshot_queries`) uses `MAILSHOTS_REMOTE_MYSQL_DB`.
* Source-data query execution (built-in/custom recipient sources) uses `MEMBERS_REMOTE_MYSQL_DB`.
* Source-data query execution for membership built-in/custom recipient sources uses `MEMBERS_REMOTE_MYSQL_DB`.
* Source-data query execution for FEN editorial, advertising, and invoice built-in sources uses `FEN_REMOTE_MYSQL_DB`.
## Problem with prior approach
@ -26,7 +28,7 @@ This blocks non-expert and non-admin users.
Use a strict, constrained query language (DSL) for mailshot audience selection.
The DSL text is stored as the data-source definition in `MAILSHOTS_REMOTE_MYSQL_DB.mailshot_queries`, then parsed and compiled into safe SQL executed against approved source-data tables in `MEMBERS_REMOTE_MYSQL_DB`.
The DSL text is stored as the data-source definition in `MAILSHOTS_REMOTE_MYSQL_DB.mailshot_queries`, then parsed and compiled into safe SQL executed against approved source-data tables in `MEMBERS_REMOTE_MYSQL_DB` and `FEN_REMOTE_MYSQL_DB`.
## Goals
@ -43,7 +45,7 @@ The DSL text is stored as the data-source definition in `MAILSHOTS_REMOTE_MYSQL_
* All pending renewals.
* All accounts where name is not in an exclusion table.
## Strict grammar (v1.5)
## Strict grammar (v1.6)
Use lowercase keywords only.
@ -57,7 +59,15 @@ source_term = source_ref | "(" , source_expr , ")" ;
source_op = "and" ;
source_ref = built_in_source | table_ref ;
built_in_source = "contacts" | "accounts" | "renewals" ;
built_in_source = "contacts"
| "accounts"
| "renewals"
| "advertisers"
| "ads"
| "pages"
| "articles"
| "issues"
| "invoices" ;
table_ref = ident_part , "." , ident_part ;
ident_part = identifier | quoted_identifier ;
@ -78,8 +88,14 @@ filter_expr = filter_name
filter_name = "selected-renewal"
| "pending-renewal"
| "selected"
| "page-in-issue"
| "ad-in-issue"
| "pending-invoice"
| "selected-invoice"
| "invoice-ids"
| "fen1-contact"
| "fen2-contact"
| "primary-contact"
| "member-or-affiliate-or-parish-council" ;
field_ref = source_ref , "." , field_name ;
@ -97,7 +113,7 @@ letter = "a"…"z" | "A"…"Z" ;
digit = "0"…"9" ;
```
## Semantics (v1.5)
## Semantics (v1.6)
* `built_in_source` maps to a predefined table alias.
* For `accounts`, the compiler must implicitly left-join account picklists so additional readable virtual fields are available:
@ -118,10 +134,40 @@ digit = "0"…"9" ;
* For custom sources, an explicit field equality predicate can provide join semantics.
* `where` applies after source composition.
* `not` negates only the next predicate/group.
* Predefined predicates are `selected-renewal`, `pending-renewal`, `fen1-contact`, `fen2-contact`, and `member-or-affiliate-or-parish-council`.
* Predefined predicates are `selected-renewal`, `pending-renewal`, `selected`, `page-in-issue`, `ad-in-issue`, `pending-invoice`, `selected-invoice`, `invoice-ids`, `fen1-contact`, `primary-contact`, and `member-or-affiliate-or-parish-council`.
* `renewals` is a built-in source mapped to membership renewal rows.
* `pending-renewal` applies only when source set includes `renewals` and means `renewals.status = 'pending'`.
* `selected-renewal` applies only when source set includes `renewals` and means `renewals.selected = true`.
* `primary-contact` applies only when source set includes `contacts` and means `contacts.is_contact_1` is truthy.
* FEN editorial, advertising, and invoice data must be exposed as built-in sources, not custom table references, using the source names below.
* These built-in sources are derived from the sibling `../feca2-app/server/src/lib/mailshotDsl.js` implementation, except that `articles` must now be promoted to a first-class built-in source. In the sibling implementation, `Articles` is used by an issue-scoped article filter but is not listed as a standalone built-in source.
* FEN built-in source table mappings:
* `advertisers` maps to `fen.advertisers`.
* `ads` maps to `fen.ads`.
* `pages` maps to `fen.Pages`.
* `articles` maps to `fen.Articles`.
* `issues` maps to `fen.Issues`.
* `invoices` maps to `fen.invoices`.
* Required FEN source fields and canonical token aliases:
* `advertisers`: `name` / `advertisername` from `AdvertiserName`.
* `ads`: `id` from `ID`, `advertiser` from `Advertiser`, `adsize` / `size` from `AdSize`, `price` from `Price`, `issue` from the related page `Issue`, `pageid` from `PageID`, `state` from `State`, `notes` from `Notes`.
* `pages`: `id` from `ID`, `issue` from `Issue`, `page` from `Page`.
* `articles`: `id` from `ID`, `pageid` from `PageID`, `articlenumber` / `article` from `ArticleNumber`, `content` from `Content`, `membername` from `MemberName`, `author` from `Author`, `dcn` from `DCN`, `articlewords` from `ArticleWords`, `otherwords` from `OtherWords`, `othercontent` / `other` from `OtherContent`.
* `issues`: `id` / `issue` from `ID`, `issuemonths` from `IssueMonths`, `description` from `Description`.
* `invoices`: `id`, `issue` / `issue_id`, `ad_id`, `invoice_number`, `invoice_date`, `due_date`, `invoice_page`, `invoice_size`, `invoice_price`, `status`, `payment_date`, `amount_paid`, `payment_method`, `payment_reference`, `notes`, `created_at`, `updated_at`.
* Required FEN physical columns:
* `fen.Issues`: `ID`, `CopyDate`, `PublicationDate`, `IssueMonths`, `Description`.
* `fen.Pages`: `ID`, `Issue`, `Page`, `PageSizeName`, `Content`.
* `fen.Articles`: `ID`, `PageID`, `ArticleNumber`, `Content`, `MemberName`, `Author`, `DCN`, `ArticleWords`, `OtherWords`, `OtherContent`.
* `fen.ads`: `ID`, `PageID`, `AdSize`, `Advertiser`, `Price`, `State`, `Notes`.
* `fen.advertisers`: `Entry ID`, `AdvertiserName`, `title`, `contact_name`, `address_1`, `address_2`, `town`, `post_code`, `Description`, `IsLapsed?`, `Home Phone`, `Phone`, `Email`, `Selected`.
* `fen.invoices`: `id`, `issue_id`, `ad_id`, `invoice_number`, `invoice_date`, `due_date`, `invoice_page`, `invoice_size`, `invoice_price`, `status`, `payment_date`, `amount_paid`, `payment_method`, `payment_reference`, `notes`, `created_at`, `updated_at`.
* `selected` applies only when source set includes `advertisers` and means `advertisers.Selected` is truthy.
* `page-in-issue(<issue>)` accepts exactly one numeric issue ID. It applies only when the source set includes `pages` or `articles`, and means the page row, or the page joined from the article row, belongs to that issue.
* `ad-in-issue(<issue>)` accepts exactly one numeric issue ID. It applies only when the source set includes one of `advertisers`, `ads`, `pages`, `issues`, or `invoices`, and means the row has an ad in that issue.
* `pending-invoice` applies only when source set includes `invoices` and means `invoices.status = 'pending'`.
* `selected-invoice` applies only when source set includes `invoices` and means the invoice ID is in the runtime selected invoice ID list.
* `invoice-ids(...)` applies only when source set includes `invoices`; it accepts one or more numeric invoice IDs.
* `member-or-affiliate-or-parish-council` applies to account data and means:
* `accounts.type` is `Member` or `Affiliate`, or
* `accounts.name` contains `Parish Council`.
@ -133,10 +179,11 @@ digit = "0"…"9" ;
* `field_ref not in (field_ref)`
* For `in(field_ref)` / `not in(field_ref)`, the RHS source is treated as a reference source (subquery semantics), not a joined source.
* `contains`, `starts-with`, and `ends-with` require a literal right-hand side value (not a field reference).
* In v1.5, mixing built-in sources and custom sources in the same sentence is not supported; validation must fail with a clear error.
* In v1.6, mixing built-in sources and custom sources in the same sentence is not supported; validation must fail with a clear error.
* For multi-custom-source sentences, all selected custom sources must be connected by explicit `=` field-to-field predicates (graph-connected join semantics), otherwise validation fails.
* `in(field_ref)` / `not in(field_ref)` must compile as subquery membership; the RHS source is a reference source and does not need to appear in `source_expr`.
* Canonical naming in this document uses exact field names as defined in database metadata for the selected source. The implementation must reject invalid names and must not add aliasing/normalization fallback.
* Canonical naming for built-in sources uses the token aliases listed in this document.
* Canonical naming for custom table references uses exact field names as defined in database metadata for the selected source. The implementation must reject invalid custom field names and must not add aliasing/normalization fallback.
### Join-path model (required)
@ -148,6 +195,20 @@ The compiler must use an explicit join graph per source pair. Example v1 join pa
* `accounts` -> `renewals`: `accounts.ID = renewals.account_id`
* `renewals` -> `contacts`: `renewals.account_id = contacts.Accountid`
* `contacts` -> `renewals`: `contacts.Accountid = renewals.account_id`
* `articles` -> `pages`: `articles.PageID = pages.ID`
* `pages` -> `articles`: `pages.ID = articles.PageID`
* `pages` -> `issues`: `pages.Issue = issues.ID`
* `issues` -> `pages`: `issues.ID = pages.Issue`
* `ads` -> `pages`: `ads.PageID = pages.ID`
* `pages` -> `ads`: `pages.ID = ads.PageID`
* `ads` -> `advertisers`: normalized `ads.Advertiser = advertisers.AdvertiserName`
* `advertisers` -> `ads`: normalized `advertisers.AdvertiserName = ads.Advertiser`
* `invoices` -> `ads`: `invoices.ad_id = ads.ID`
* `ads` -> `invoices`: `ads.ID = invoices.ad_id`
* `invoices` -> `issues`: `invoices.issue_id = issues.ID`
* `issues` -> `invoices`: `issues.ID = invoices.issue_id`
* `invoices` -> `pages`: `invoices.issue_id = pages.Issue`
* `pages` -> `invoices`: `pages.Issue = invoices.issue_id`
If no approved join path exists between two sources for `and`, parsing/validation must fail with a clear error.
@ -160,7 +221,7 @@ No implicit join behavior is allowed:
* `contacts`
* `contacts and accounts where fen1-contact`
* `contacts and accounts where fen2-contact`
* `contacts and accounts where primary-contact`
* `accounts and contacts where contacts.Accountid = accounts.ID`
* `contacts where contacts.Last contains 'smith'`
* `contacts and accounts where accounts.Type = 'Member' and contacts.FENContact1 = true`
@ -168,6 +229,11 @@ No implicit join behavior is allowed:
* `renewals where pending-renewal`
* `renewals where selected-renewal`
* `renewals and accounts and contacts where pending-renewal`
* `advertisers where selected`
* `ads and pages where ad-in-issue(202605)`
* `articles and pages where page-in-issue(202605)`
* `invoices where pending-invoice`
* `invoices and ads and advertisers where invoice-ids(101, 102)`
* `accounts where accounts.Name not in (members.ExcludedAccounts.ExcludedAccount)`
* `members.mailshot_test`
@ -179,6 +245,12 @@ No implicit join behavior is allowed:
* `contacts or accounts` (`or` is not supported; use `and` only)
* `accounts where pending-renewal` (invalid: filter requires `renewals` source)
* `accounts where fen1-contact` (invalid: filter requires `contacts` source)
* `accounts where primary-contact` (invalid: filter requires `contacts` source)
* `ads where selected` (invalid: filter requires `advertisers` source)
* `advertisers where pending-invoice` (invalid: filter requires `invoices` source)
* `pages where ad-in-issue` (invalid: `ad-in-issue` requires exactly one numeric issue ID)
* `ads where page-in-issue(202605)` (invalid: `page-in-issue` requires `pages` or `articles` source)
* `invoices where invoice-ids('abc')` (invalid: `invoice-ids` accepts numeric invoice IDs only)
* `renewals where member-or-affiliate-or-parish-council` (invalid: filter requires `accounts` source)
* `accounts and contacts where contacts.Accountid contains accounts.ID` (invalid: field-to-field supports only `=`/`!=`)
* `accounts where accounts.Name contains members.ExcludedAccounts.ExcludedAccount` (invalid: `contains` requires a literal RHS)
@ -345,8 +417,14 @@ Predefined filter control:
* examples:
* `Renewal is selected` -> `selected-renewal`
* `Renewal is pending` -> `pending-renewal`
* `Advertiser is selected` -> `selected`
* `Page is in issue` -> `page-in-issue(...)`
* `Advertiser has ad in issue` -> `ad-in-issue(...)`
* `Invoice is pending` -> `pending-invoice`
* `Invoice is selected` -> `selected-invoice`
* `Invoice ID is one of` -> `invoice-ids(...)`
* `Contact is FEN1` -> `fen1-contact`
* `Contact is FEN2` -> `fen2-contact`
* `Contact is primary` -> `primary-contact`
* `Account is member/affiliate/parish council` -> `member-or-affiliate-or-parish-council`
Field comparison row:
@ -361,7 +439,7 @@ Field comparison row:
Logical composition:
* all rows combine with `and` in v1.5
* all rows combine with `and` in v1.6
* each row can be negated via checkbox (`not`)
* optional group rows allow nested bracketed expressions

View File

@ -31,11 +31,13 @@ Account details:
* `accounts.id`
* `accounts.name`
* `accounts.account_audience`
* `accounts.account_no_of_members`
* `accounts.account_type_id` plus joined `picklist_account_type.value` exposed as `accounts.type`
* `accounts.public_location_id` plus joined `picklist_public_location.value` exposed as `accounts.public_location`
* `accounts.sector_id` plus joined `picklist_sector.value` exposed as `accounts.account_sector`
* `accounts.facebook`
* `accounts.twitter`
* `accounts.other_social`
Renewal details:
@ -56,16 +58,16 @@ Contact 1 details (`contacts.is_contact_1 = 1`):
* Position/title
* Address
* Email
* Phone
* Mobile
* Phone 1
* Phone 2
Contact 2 details (`contacts.is_contact_2 = 1`):
* Name
* Position/title
* Email
* Phone
* Mobile
* Phone 1
* Phone 2
Fen Edge News contact details (`contacts.is_fen_1 = 1`):
@ -97,7 +99,6 @@ Required columns:
* `account_id` -> `accounts.id`
* `account_name` -> `accounts.name`
* `account_audience` -> `accounts.account_audience`
* `account_no_of_members` -> `accounts.account_no_of_members`
* `account_type_id` -> `accounts.account_type_id`
* `account_type` -> `picklist_account_type.value` (equivalent to `accounts.type`)
@ -105,6 +106,9 @@ Required columns:
* `account_sector` -> `picklist_sector.value` (equivalent to `accounts.account_sector`)
* `public_location_id` -> `accounts.public_location_id`
* `public_location` -> `picklist_public_location.value` (equivalent to `accounts.public_location`)
* `account_facebook` -> `accounts.facebook`
* `account_twitter` -> `accounts.twitter`
* `account_other_social` -> `accounts.other_social`
Filter:
@ -124,7 +128,7 @@ Minimum required prefixed columns:
* `..._contact_id` -> `contacts.id`
* `..._account_id` -> `contacts.account_id`
* `..._name` -> `TRIM(CONCAT_WS(' ', contacts.first_name, contacts.middle_name, contacts.last_name))`
* `..._name` -> `TRIM(CONCAT_WS(' ', contacts.first_name, contacts.last_name))`
* `..._position` -> `contacts.title`
* `..._address_1` -> `contacts.contact_address_1`
* `..._address_2` -> `contacts.contact_address_2`
@ -132,8 +136,8 @@ Minimum required prefixed columns:
* `..._county` -> `contacts.contact_county`
* `..._postcode` -> `contacts.contact_postcode`
* `..._email` -> `contacts.contact_email_1`
* `..._phone` -> `contacts.home_phone`
* `..._mobile` -> `contacts.mobile`
* `..._phone_1` -> `contacts.phone_1`
* `..._phone_2` -> `contacts.phone_2`
### `renewal_public_contact`
@ -207,15 +211,14 @@ The following names in prior drafts do not match current schema and are explicit
* `members.renewals` -> `NON-EXISTENT` in that schema name (use `fenedgec_members.renewals`)
* `Account.Name` -> `NON-EXISTENT` (use `accounts.name`)
* `Account.Type` -> `NON-EXISTENT` (use `accounts.account_type_id` + `picklist_account_type.slug` for member-status checks, and `picklist_account_type.value` for display)
* `Account.AccountAudience` -> `NON-EXISTENT` (use `accounts.account_audience`)
* `Account.AccountNoOfMembers` -> `NON-EXISTENT` (use `accounts.account_no_of_members`)
* `Contact.Accountid` -> `NON-EXISTENT` (use `contacts.account_id`)
* `First`/`Middle`/`Last` -> `NON-EXISTENT` (use `first_name`/`middle_name`/`last_name`)
* `First`/`Middle`/`Last` -> `NON-EXISTENT` (use `first_name`/`last_name`)
* `JobTitle` -> `NON-EXISTENT`; mapped to `contacts.title` (`AMBIGUOUS: verify business meaning`)
* `Address1`/`Address2`/`Town`/`County`/`Postcode` -> `NON-EXISTENT` (use `contact_address_1`/`contact_address_2`/`contact_town`/`contact_county`/`contact_postcode`)
* `Email` -> `NON-EXISTENT`; mapped to `contacts.contact_email_1` (`AMBIGUOUS: confirm whether `contact_email_2 ` should also be included`)
* `Home` -> `NON-EXISTENT` (use `contacts.home_phone`)
* `Mobile` -> `NON-EXISTENT` (use `contacts.mobile`)
* `Home` -> `NON-EXISTENT` (use `contacts.phone_1`)
* `Mobile` -> `NON-EXISTENT` (use `contacts.phone_2`)
* `Contact1` -> `NON-EXISTENT` (use `contacts.is_contact_1`)
* `Contact2` -> `NON-EXISTENT` (use `contacts.is_contact_2`)
* `FENContact1` -> `NON-EXISTENT` (use `contacts.is_fen_1`)
@ -231,8 +234,9 @@ The following names in prior drafts do not match current schema and are explicit
* `renewal_fen_contact` only includes contacts flagged `is_fen_1 = 1`.
* `renewal_public_contact` only includes contacts flagged `is_public_contact = 1`.
* `renewal_accounts_with_contacts` returns one row per pending renewal row (no duplicated `renewal_id`).
* `account_audience` and `account_no_of_members` are present in final output.
* `account_no_of_members` is present in final output.
* Renewal fields listed above are present in final output.
* Account picklist-backed values are present in final output as `account_type`/`account_sector`/`public_location` (or equivalent `accounts.type`/`accounts.account_sector`/`accounts.public_location` aliases).
* Account social fields are present in final output as `account_facebook`, `account_twitter`, and `account_other_social`.
* Member-status check is based on `picklist_account_type.slug`, not the label text.
* No remote cross-system query is used by these views.

View File

@ -15,14 +15,16 @@ CREATE VIEW `fenedgec_members`.`renewal_members` AS
SELECT
a.`id` AS `account_id`,
a.`name` AS `account_name`,
a.`account_audience` AS `account_audience`,
a.`account_no_of_members` AS `account_no_of_members`,
a.`account_type_id` AS `account_type_id`,
pat.`value` AS `account_type`,
a.`sector_id` AS `sector_id`,
ps.`value` AS `account_sector`,
a.`public_location_id` AS `public_location_id`,
ppl.`value` AS `public_location`
ppl.`value` AS `public_location`,
a.`facebook` AS `account_facebook`,
a.`twitter` AS `account_twitter`,
a.`other_social` AS `account_other_social`
FROM `fenedgec_members`.`accounts` a
LEFT JOIN `fenedgec_members`.`picklist_account_type` pat
ON pat.`id` = a.`account_type_id`
@ -52,7 +54,7 @@ CREATE VIEW `fenedgec_members`.`renewal_contact_1` AS
SELECT
c.`id` AS `contact_1_contact_id`,
c.`account_id` AS `contact_1_account_id`,
TRIM(CONCAT_WS(' ', c.`first_name`, c.`middle_name`, c.`last_name`)) AS `contact_1_name`,
TRIM(CONCAT_WS(' ', c.`first_name`, c.`last_name`)) AS `contact_1_name`,
c.`title` AS `contact_1_position`,
c.`contact_address_1` AS `contact_1_address_1`,
c.`contact_address_2` AS `contact_1_address_2`,
@ -60,8 +62,8 @@ SELECT
c.`contact_county` AS `contact_1_county`,
c.`contact_postcode` AS `contact_1_postcode`,
c.`contact_email_1` AS `contact_1_email`,
c.`home_phone` AS `contact_1_phone`,
c.`mobile` AS `contact_1_mobile`
c.`phone_1` AS `contact_1_phone_1`,
c.`phone_2` AS `contact_1_phone_2`
FROM `fenedgec_members`.`contacts` c
INNER JOIN (
SELECT `account_id`, MIN(`id`) AS `min_id`
@ -79,7 +81,7 @@ CREATE VIEW `fenedgec_members`.`renewal_contact_2` AS
SELECT
c.`id` AS `contact_2_contact_id`,
c.`account_id` AS `contact_2_account_id`,
TRIM(CONCAT_WS(' ', c.`first_name`, c.`middle_name`, c.`last_name`)) AS `contact_2_name`,
TRIM(CONCAT_WS(' ', c.`first_name`, c.`last_name`)) AS `contact_2_name`,
c.`title` AS `contact_2_position`,
c.`contact_address_1` AS `contact_2_address_1`,
c.`contact_address_2` AS `contact_2_address_2`,
@ -87,8 +89,8 @@ SELECT
c.`contact_county` AS `contact_2_county`,
c.`contact_postcode` AS `contact_2_postcode`,
c.`contact_email_1` AS `contact_2_email`,
c.`home_phone` AS `contact_2_phone`,
c.`mobile` AS `contact_2_mobile`
c.`phone_1` AS `contact_2_phone_1`,
c.`phone_2` AS `contact_2_phone_2`
FROM `fenedgec_members`.`contacts` c
INNER JOIN (
SELECT `account_id`, MIN(`id`) AS `min_id`
@ -106,7 +108,7 @@ CREATE VIEW `fenedgec_members`.`renewal_fen_contact` AS
SELECT
c.`id` AS `fen_contact_contact_id`,
c.`account_id` AS `fen_contact_account_id`,
TRIM(CONCAT_WS(' ', c.`first_name`, c.`middle_name`, c.`last_name`)) AS `fen_contact_name`,
TRIM(CONCAT_WS(' ', c.`first_name`, c.`last_name`)) AS `fen_contact_name`,
c.`contact_email_1` AS `fen_contact_email`
FROM `fenedgec_members`.`contacts` c
INNER JOIN (
@ -125,7 +127,7 @@ CREATE VIEW `fenedgec_members`.`renewal_public_contact` AS
SELECT
c.`id` AS `public_contact_contact_id`,
c.`account_id` AS `public_contact_account_id`,
TRIM(CONCAT_WS(' ', c.`first_name`, c.`middle_name`, c.`last_name`)) AS `public_contact_name`,
TRIM(CONCAT_WS(' ', c.`first_name`, c.`last_name`)) AS `public_contact_name`,
c.`public_phone` AS `public_contact_public_phone`,
c.`public_email` AS `public_contact_public_email`
FROM `fenedgec_members`.`contacts` c
@ -155,8 +157,8 @@ SELECT
rc1.`contact_1_county`,
rc1.`contact_1_postcode`,
rc1.`contact_1_email`,
rc1.`contact_1_phone`,
rc1.`contact_1_mobile`,
rc1.`contact_1_phone_1`,
rc1.`contact_1_phone_2`,
rc2.`contact_2_contact_id`,
rc2.`contact_2_account_id`,
rc2.`contact_2_name`,
@ -167,8 +169,8 @@ SELECT
rc2.`contact_2_county`,
rc2.`contact_2_postcode`,
rc2.`contact_2_email`,
rc2.`contact_2_phone`,
rc2.`contact_2_mobile`,
rc2.`contact_2_phone_1`,
rc2.`contact_2_phone_2`,
rcf.`fen_contact_contact_id`,
rcf.`fen_contact_account_id`,
rcf.`fen_contact_name`,