From e02696ea2f9be975507277dc2b71bb211f2db6b1 Mon Sep 17 00:00:00 2001 From: Adrian Stephens Date: Sat, 9 May 2026 07:28:18 +0100 Subject: [PATCH] preparing for fen db --- .../feca_mailshots_plugin.php | 2 +- .../src/Application/DataSourceService.php | 72 ---- .../src/Application/DslCompiler.php | 4 +- .../src/Application/MailshotRunService.php | 91 +---- .../src/Application/TemplateRenderer.php | 9 +- formats/renewals_v5.html | 385 ++++++++++++++++++ requirements/mailshot_data_source.md | 102 ++++- requirements/renewal_views.md | 32 +- scripts/sql/create_renewal_views.sql | 30 +- 9 files changed, 538 insertions(+), 189 deletions(-) create mode 100644 formats/renewals_v5.html diff --git a/feca_mailshots_plugin/feca_mailshots_plugin.php b/feca_mailshots_plugin/feca_mailshots_plugin.php index 6199bce..9c78f2f 100644 --- a/feca_mailshots_plugin/feca_mailshots_plugin.php +++ b/feca_mailshots_plugin/feca_mailshots_plugin.php @@ -3,7 +3,7 @@ * Plugin Name: FECA Mailshots * Plugin URI: https://fenedge.co.uk/ * Description: FECA mailshots plugin. - * Version: 1.0.0 + * Version: 1.0.3 * Requires at least: 6.0 * Requires PHP: 7.4 * Author: FECA diff --git a/feca_mailshots_plugin/src/Application/DataSourceService.php b/feca_mailshots_plugin/src/Application/DataSourceService.php index e7a520d..b8e3c44 100644 --- a/feca_mailshots_plugin/src/Application/DataSourceService.php +++ b/feca_mailshots_plugin/src/Application/DataSourceService.php @@ -145,12 +145,8 @@ final class DataSourceService $stmtRows = $this->router->membersPdo()->prepare($previewSql); $stmtRows->execute($compiled['params']); $rows = $stmtRows->fetchAll(PDO::FETCH_ASSOC); - $rows = $this->withExpectedFieldAliases($rows, $validation['expected_fields']); $columnSet = []; - foreach ($validation['expected_fields'] as $field) { - $columnSet[$field] = true; - } foreach ($rows as $row) { foreach (array_keys($row) as $key) { $columnSet[$key] = true; @@ -167,74 +163,6 @@ final class DataSourceService ]; } - /** - * Ensure preview rows expose the same qualified field names used by token generation. - * - * @param list> $rows - * @param list $expectedFields - * @return list> - */ - private function withExpectedFieldAliases(array $rows, array $expectedFields): array - { - if ($rows === [] || $expectedFields === []) { - return $rows; - } - - $expectedByLeaf = []; - foreach ($expectedFields as $qualified) { - $qualified = trim((string) $qualified); - if ($qualified === '') { - continue; - } - $leaf = $this->leafFieldName($qualified); - if ($leaf === '') { - continue; - } - $expectedByLeaf[strtolower($leaf)][] = $qualified; - } - - if ($expectedByLeaf === []) { - return $rows; - } - - foreach ($rows as &$row) { - if (!is_array($row) || $row === []) { - continue; - } - - $keyLookup = []; - foreach (array_keys($row) as $key) { - $k = (string) $key; - $keyLookup[strtolower($k)] = $k; - } - - foreach ($expectedByLeaf as $leafLower => $qualifiedList) { - if (!isset($keyLookup[$leafLower])) { - continue; - } - $sourceKey = $keyLookup[$leafLower]; - foreach ($qualifiedList as $qualified) { - if (array_key_exists($qualified, $row)) { - continue; - } - $row[$qualified] = $row[$sourceKey]; - } - } - } - unset($row); - - return $rows; - } - - private function leafFieldName(string $qualifiedField): string - { - $pos = strrpos($qualifiedField, '.'); - if ($pos === false) { - return $qualifiedField; - } - return substr($qualifiedField, $pos + 1); - } - /** @return array> */ public function sourceFields(): array { diff --git a/feca_mailshots_plugin/src/Application/DslCompiler.php b/feca_mailshots_plugin/src/Application/DslCompiler.php index 0e3ff23..101f585 100644 --- a/feca_mailshots_plugin/src/Application/DslCompiler.php +++ b/feca_mailshots_plugin/src/Application/DslCompiler.php @@ -57,7 +57,9 @@ final class DslCompiler $whereSql = $whereParts === [] ? '' : ' WHERE ' . implode(' AND ', $whereParts); - $selectSql = count($sources) > 1 ? $this->buildUniqueSelectProjection($sources) : '*'; + // Always emit source-qualified projection keys so token names remain stable + // without any post-query alias fallback. + $selectSql = $this->buildUniqueSelectProjection($sources); $sql = 'SELECT ' . $selectSql . ' FROM ' . $from; if ($joins !== []) { $sql .= ' ' . implode(' ', $joins); diff --git a/feca_mailshots_plugin/src/Application/MailshotRunService.php b/feca_mailshots_plugin/src/Application/MailshotRunService.php index b310760..9536968 100644 --- a/feca_mailshots_plugin/src/Application/MailshotRunService.php +++ b/feca_mailshots_plugin/src/Application/MailshotRunService.php @@ -265,6 +265,11 @@ final class MailshotRunService /** @return array */ public function runMailshot(int $mailshotId): array { + $mailshot = $this->mailshots->find($mailshotId); + if ($mailshot === null) { + return ['ok' => false, 'errors' => ['Mailshot not found.']]; + } + $creds = $this->credentials->credentials(); if ($creds === null) { return ['ok' => false, 'errors' => ['Missing mail credentials. Configure FECA Mailshots Profile page first.']]; @@ -396,10 +401,14 @@ final class MailshotRunService } $files = []; - $mergedHtmlTmpPath = null; - $mergedHtmlHandle = null; - $ghostscriptBinary = $this->findExecutableBinary('gs', ['/usr/bin/gs', '/bin/gs']); - $useGhostscriptMerge = $includeMerged && $ghostscriptBinary !== '' && function_exists('exec'); + $ghostscriptBinary = $this->findExecutableBinary('gs'); + if ($includeMerged && $ghostscriptBinary === '') { + return ['ok' => false, 'errors' => ['Merged PDF generation requires Ghostscript (gs) to be installed and available on PATH.']]; + } + if ($includeMerged && !function_exists('exec')) { + return ['ok' => false, 'errors' => ['Merged PDF generation requires PHP exec() to be enabled.']]; + } + $useGhostscriptMerge = $includeMerged; $mergedPdfTmpDir = ''; $mergedPdfPartPaths = []; $nameCounts = []; @@ -412,28 +421,6 @@ final class MailshotRunService if (!@mkdir($mergedPdfTmpDir, 0700, true) && !is_dir($mergedPdfTmpDir)) { return ['ok' => false, 'errors' => ['Unable to prepare temporary directory for merged PDF build.']]; } - } elseif ($includeMerged) { - $tmpBasePath = tempnam(sys_get_temp_dir(), 'feca_mailshots_merged_html_'); - if (!is_string($tmpBasePath) || $tmpBasePath === '') { - return ['ok' => false, 'errors' => ['Unable to allocate temporary merged HTML file.']]; - } - $mergedHtmlTmpPath = $tmpBasePath . '.html'; - if (!@rename($tmpBasePath, $mergedHtmlTmpPath)) { - @unlink($tmpBasePath); - return ['ok' => false, 'errors' => ['Unable to prepare temporary merged HTML file.']]; - } - $mergedHtmlHandle = @fopen($mergedHtmlTmpPath, 'wb'); - if (!is_resource($mergedHtmlHandle)) { - @unlink($mergedHtmlTmpPath); - return ['ok' => false, 'errors' => ['Unable to open temporary merged HTML file for writing.']]; - } - fwrite( - $mergedHtmlHandle, - '' - ); } foreach (array_values($rows) as $index => $row) { @@ -468,9 +455,6 @@ final class MailshotRunService $mergedPdfPartPaths[] = $partPath; $mergedSectionCount++; unset($pdfBytesForMerge); - } elseif ($includeMerged) { - fwrite($mergedHtmlHandle, '
' . $pdfHtml . '
'); - $mergedSectionCount++; } unset($rendered, $pdfHtml); } catch (\Throwable $e) { @@ -482,57 +466,28 @@ final class MailshotRunService } } - if ($includeMerged && is_resource($mergedHtmlHandle)) { - fwrite($mergedHtmlHandle, ''); - fclose($mergedHtmlHandle); - $mergedHtmlHandle = null; - } - if ($errors !== []) { - if ($mergedHtmlTmpPath !== null) { - @unlink($mergedHtmlTmpPath); - } $this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths); return ['ok' => false, 'errors' => $errors]; } if ($includeFiles && $files === []) { - if ($mergedHtmlTmpPath !== null) { - @unlink($mergedHtmlTmpPath); - } $this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths); return ['ok' => false, 'errors' => ['No PDF attachments were generated from this mailshot.']]; } if ($includeMerged && $mergedSectionCount === 0) { - if ($mergedHtmlTmpPath !== null) { - @unlink($mergedHtmlTmpPath); - } $this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths); return ['ok' => false, 'errors' => ['No merged PDF content was generated from this mailshot.']]; } $mergedPdfBytes = ''; if ($includeMerged) { - if ($useGhostscriptMerge) { - try { - $mergedPdfBytes = $this->mergePdfFilesWithGhostscript($ghostscriptBinary, $mergedPdfPartPaths); - } catch (\Throwable $e) { - $this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths); - return ['ok' => false, 'errors' => ['Merged PDF generation failed: ' . $e->getMessage()]]; - } + try { + $mergedPdfBytes = $this->mergePdfFilesWithGhostscript($ghostscriptBinary, $mergedPdfPartPaths); + } catch (\Throwable $e) { $this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths); - } else { - try { - $mergedPdfBytes = $this->renderPdfBytesFromHtmlFile((string) $mergedHtmlTmpPath); - } catch (\Throwable $e) { - if ($mergedHtmlTmpPath !== null) { - @unlink($mergedHtmlTmpPath); - } - return ['ok' => false, 'errors' => ['Merged PDF generation failed: ' . $e->getMessage()]]; - } - if ($mergedHtmlTmpPath !== null) { - @unlink($mergedHtmlTmpPath); - } + return ['ok' => false, 'errors' => ['Merged PDF generation failed: ' . $e->getMessage()]]; } + $this->cleanupMergedPartFiles($mergedPdfTmpDir, $mergedPdfPartPaths); } return [ @@ -549,7 +504,7 @@ final class MailshotRunService public function generatePdfZipToTemp(int $mailshotId): array { $canUseZipArchive = class_exists('ZipArchive'); - $zipBinary = $this->findExecutableBinary('zip', ['/usr/bin/zip', '/bin/zip']); + $zipBinary = $this->findExecutableBinary('zip'); if (!$canUseZipArchive && $zipBinary === '') { return ['ok' => false, 'errors' => ['Neither ZipArchive extension nor zip CLI binary is available.']]; } @@ -707,7 +662,7 @@ final class MailshotRunService } } - private function findExecutableBinary(string $binaryName, array $fallbackPaths = []): string + private function findExecutableBinary(string $binaryName): string { $pathEnv = (string) getenv('PATH'); $candidates = []; @@ -719,9 +674,6 @@ final class MailshotRunService } } } - foreach ($fallbackPaths as $p) { - $candidates[] = $p; - } foreach ($candidates as $candidate) { if (is_string($candidate) && $candidate !== '' && is_file($candidate) && is_executable($candidate)) { @@ -932,8 +884,9 @@ final class MailshotRunService } } } + return null; } - foreach (['Email', 'email', 'contact_email', 'account_email', 'recipient_email', 'recipient_email_last'] as $key) { + foreach (['Email', 'email', 'contact_email', 'recipient_email', 'recipient_email_last'] as $key) { if (isset($row[$key])) { $v = trim((string) $row[$key]); if ($v !== '' && filter_var($v, FILTER_VALIDATE_EMAIL)) { diff --git a/feca_mailshots_plugin/src/Application/TemplateRenderer.php b/feca_mailshots_plugin/src/Application/TemplateRenderer.php index 39087a5..87a5c82 100644 --- a/feca_mailshots_plugin/src/Application/TemplateRenderer.php +++ b/feca_mailshots_plugin/src/Application/TemplateRenderer.php @@ -143,10 +143,10 @@ final class TemplateRenderer */ private function transcodePngToJpeg(string $assetName, string $pngBytes): array { - $convertBinary = $this->findExecutableBinary('convert', ['/usr/bin/convert', '/bin/convert']); + $convertBinary = $this->findExecutableBinary('convert'); if ($convertBinary === '') { throw new \RuntimeException( - 'pdf_asset("' . $assetName . '") uses PNG, but this runtime lacks GD and ImageMagick convert.' + 'pdf_asset("' . $assetName . '") uses PNG, but neither GD PNG support nor ImageMagick convert is available.' ); } if (!function_exists('exec')) { @@ -199,7 +199,7 @@ final class TemplateRenderer } } - private function findExecutableBinary(string $binaryName, array $fallbackPaths = []): string + private function findExecutableBinary(string $binaryName): string { $pathEnv = (string) getenv('PATH'); $candidates = []; @@ -211,9 +211,6 @@ final class TemplateRenderer } } } - foreach ($fallbackPaths as $p) { - $candidates[] = $p; - } foreach ($candidates as $candidate) { if (is_string($candidate) && $candidate !== '' && is_file($candidate) && is_executable($candidate)) { diff --git a/formats/renewals_v5.html b/formats/renewals_v5.html new file mode 100644 index 0000000..5fabfce --- /dev/null +++ b/formats/renewals_v5.html @@ -0,0 +1,385 @@ + + + +
DRAFT
+
+
+
{{ pdf_asset("Fen Edge Logo") }}
+
+

Membership Renewal 2026-2027

+

Fen Edge Community Association

+
+
Group Membership Fee: GBP15
+

Please amend any details that have changed before returning this form.

+
+

+
+ +
+
Account And Public Contact
+
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + +
Account{{ fenedgec_members_renewal_accounts_with_contacts_account_name }}
Type{{ fenedgec_members_renewal_accounts_with_contacts_account_type }}
Sector{{ fenedgec_members_renewal_accounts_with_contacts_account_sector }}
Public Location{{ fenedgec_members_renewal_accounts_with_contacts_public_location }}
Members{{ fenedgec_members_renewal_accounts_with_contacts_account_no_of_members }}
+ 		+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Public Contact{{ fenedgec_members_renewal_accounts_with_contacts_public_contact_name }}
Public Email{{ fenedgec_members_renewal_accounts_with_contacts_public_contact_public_email }}
Public Phone{{ fenedgec_members_renewal_accounts_with_contacts_public_contact_public_phone }}
Facebook{{ fenedgec_members_renewal_accounts_with_contacts_account_facebook }}
Twitter{{ fenedgec_members_renewal_accounts_with_contacts_account_twitter }}
Other Social{{ fenedgec_members_renewal_accounts_with_contacts_account_other_social }}
+
+
+
+ +
+
Primary Contact (Contact 1)
+
+ + + + + + + + + + + + + + + + + + + + + + + +
Name{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_name }}
Position{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_position }}
Address{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_address_1 }} {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_address_2 }}, {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_town }}, {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_county }}, {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_postcode }}
Email{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_email }}
Phone 1 / Phone 2{{ fenedgec_members_renewal_accounts_with_contacts_contact_1_phone_1 }} {{ fenedgec_members_renewal_accounts_with_contacts_contact_1_phone_2 }}
+
+
+ +
+
Secondary Contacts
+
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + +
Contact 2 Name{{ fenedgec_members_renewal_accounts_with_contacts_contact_2_name }}
Position{{ fenedgec_members_renewal_accounts_with_contacts_contact_2_position }}
Email{{ fenedgec_members_renewal_accounts_with_contacts_contact_2_email }}
Phone 1 / Phone 2{{ fenedgec_members_renewal_accounts_with_contacts_contact_2_phone_1 }} {{ fenedgec_members_renewal_accounts_with_contacts_contact_2_phone_2 }}
+ 		+ + + + + + + + + + + +
Fen Contact Name{{ fenedgec_members_renewal_accounts_with_contacts_fen_contact_name }}
Fen Contact Email{{ fenedgec_members_renewal_accounts_with_contacts_fen_contact_email }}
+
+
+
+ +
+ Please check your group’s public contact details on the FECA website: + www.fenedge.co.uk/full-members-list and advise any changes. +
+ + + + + + + + +
+
Signed
+

+ 		+
Date
+

+
+ +
+

Payment Options

+

1. Cheque: payable to Fen Edge Community Association, sent with this form to FECA Community Office, Cottenham Village College, Cottenham, CB24 8UA.

+

2. BACS: Sort Code 20-17-22, Account No. 40349178; then scan and email the form to info@fenedge.co.uk.

+
+
diff --git a/requirements/mailshot_data_source.md b/requirements/mailshot_data_source.md index dd3eded..b86885c 100644 --- a/requirements/mailshot_data_source.md +++ b/requirements/mailshot_data_source.md @@ -7,11 +7,13 @@ Document requirements for a safe, non-admin-friendly mailshot data source defini ## Environment baseline for this specification `members` as a database name means the value of `MEMBERS_REMOTE_MYSQL_DB`. +`fen` as a database name means the value of `FEN_REMOTE_MYSQL_DB`. For this specification: * Data-source definition persistence (create/update/delete/list of `mailshot_queries`) uses `MAILSHOTS_REMOTE_MYSQL_DB`. -* Source-data query execution (built-in/custom recipient sources) uses `MEMBERS_REMOTE_MYSQL_DB`. +* Source-data query execution for membership built-in/custom recipient sources uses `MEMBERS_REMOTE_MYSQL_DB`. +* Source-data query execution for FEN editorial, advertising, and invoice built-in sources uses `FEN_REMOTE_MYSQL_DB`. ## Problem with prior approach @@ -26,7 +28,7 @@ This blocks non-expert and non-admin users. Use a strict, constrained query language (DSL) for mailshot audience selection. -The DSL text is stored as the data-source definition in `MAILSHOTS_REMOTE_MYSQL_DB.mailshot_queries`, then parsed and compiled into safe SQL executed against approved source-data tables in `MEMBERS_REMOTE_MYSQL_DB`. +The DSL text is stored as the data-source definition in `MAILSHOTS_REMOTE_MYSQL_DB.mailshot_queries`, then parsed and compiled into safe SQL executed against approved source-data tables in `MEMBERS_REMOTE_MYSQL_DB` and `FEN_REMOTE_MYSQL_DB`. ## Goals @@ -43,7 +45,7 @@ The DSL text is stored as the data-source definition in `MAILSHOTS_REMOTE_MYSQL_ * All pending renewals. * All accounts where name is not in an exclusion table. -## Strict grammar (v1.5) +## Strict grammar (v1.6) Use lowercase keywords only. @@ -57,7 +59,15 @@ source_term = source_ref | "(" , source_expr , ")" ; source_op = "and" ; source_ref = built_in_source | table_ref ; -built_in_source = "contacts" | "accounts" | "renewals" ; +built_in_source = "contacts" + | "accounts" + | "renewals" + | "advertisers" + | "ads" + | "pages" + | "articles" + | "issues" + | "invoices" ; table_ref = ident_part , "." , ident_part ; ident_part = identifier | quoted_identifier ; @@ -78,8 +88,14 @@ filter_expr = filter_name filter_name = "selected-renewal" | "pending-renewal" + | "selected" + | "page-in-issue" + | "ad-in-issue" + | "pending-invoice" + | "selected-invoice" + | "invoice-ids" | "fen1-contact" - | "fen2-contact" + | "primary-contact" | "member-or-affiliate-or-parish-council" ; field_ref = source_ref , "." , field_name ; @@ -97,7 +113,7 @@ letter = "a"…"z" | "A"…"Z" ; digit = "0"…"9" ; ``` -## Semantics (v1.5) +## Semantics (v1.6) * `built_in_source` maps to a predefined table alias. * For `accounts`, the compiler must implicitly left-join account picklists so additional readable virtual fields are available: @@ -118,10 +134,40 @@ digit = "0"…"9" ; * For custom sources, an explicit field equality predicate can provide join semantics. * `where` applies after source composition. * `not` negates only the next predicate/group. -* Predefined predicates are `selected-renewal`, `pending-renewal`, `fen1-contact`, `fen2-contact`, and `member-or-affiliate-or-parish-council`. +* Predefined predicates are `selected-renewal`, `pending-renewal`, `selected`, `page-in-issue`, `ad-in-issue`, `pending-invoice`, `selected-invoice`, `invoice-ids`, `fen1-contact`, `primary-contact`, and `member-or-affiliate-or-parish-council`. * `renewals` is a built-in source mapped to membership renewal rows. * `pending-renewal` applies only when source set includes `renewals` and means `renewals.status = 'pending'`. * `selected-renewal` applies only when source set includes `renewals` and means `renewals.selected = true`. +* `primary-contact` applies only when source set includes `contacts` and means `contacts.is_contact_1` is truthy. +* FEN editorial, advertising, and invoice data must be exposed as built-in sources, not custom table references, using the source names below. +* These built-in sources are derived from the sibling `../feca2-app/server/src/lib/mailshotDsl.js` implementation, except that `articles` must now be promoted to a first-class built-in source. In the sibling implementation, `Articles` is used by an issue-scoped article filter but is not listed as a standalone built-in source. +* FEN built-in source table mappings: + * `advertisers` maps to `fen.advertisers`. + * `ads` maps to `fen.ads`. + * `pages` maps to `fen.Pages`. + * `articles` maps to `fen.Articles`. + * `issues` maps to `fen.Issues`. + * `invoices` maps to `fen.invoices`. +* Required FEN source fields and canonical token aliases: + * `advertisers`: `name` / `advertisername` from `AdvertiserName`. + * `ads`: `id` from `ID`, `advertiser` from `Advertiser`, `adsize` / `size` from `AdSize`, `price` from `Price`, `issue` from the related page `Issue`, `pageid` from `PageID`, `state` from `State`, `notes` from `Notes`. + * `pages`: `id` from `ID`, `issue` from `Issue`, `page` from `Page`. + * `articles`: `id` from `ID`, `pageid` from `PageID`, `articlenumber` / `article` from `ArticleNumber`, `content` from `Content`, `membername` from `MemberName`, `author` from `Author`, `dcn` from `DCN`, `articlewords` from `ArticleWords`, `otherwords` from `OtherWords`, `othercontent` / `other` from `OtherContent`. + * `issues`: `id` / `issue` from `ID`, `issuemonths` from `IssueMonths`, `description` from `Description`. + * `invoices`: `id`, `issue` / `issue_id`, `ad_id`, `invoice_number`, `invoice_date`, `due_date`, `invoice_page`, `invoice_size`, `invoice_price`, `status`, `payment_date`, `amount_paid`, `payment_method`, `payment_reference`, `notes`, `created_at`, `updated_at`. +* Required FEN physical columns: + * `fen.Issues`: `ID`, `CopyDate`, `PublicationDate`, `IssueMonths`, `Description`. + * `fen.Pages`: `ID`, `Issue`, `Page`, `PageSizeName`, `Content`. + * `fen.Articles`: `ID`, `PageID`, `ArticleNumber`, `Content`, `MemberName`, `Author`, `DCN`, `ArticleWords`, `OtherWords`, `OtherContent`. + * `fen.ads`: `ID`, `PageID`, `AdSize`, `Advertiser`, `Price`, `State`, `Notes`. + * `fen.advertisers`: `Entry ID`, `AdvertiserName`, `title`, `contact_name`, `address_1`, `address_2`, `town`, `post_code`, `Description`, `IsLapsed?`, `Home Phone`, `Phone`, `Email`, `Selected`. + * `fen.invoices`: `id`, `issue_id`, `ad_id`, `invoice_number`, `invoice_date`, `due_date`, `invoice_page`, `invoice_size`, `invoice_price`, `status`, `payment_date`, `amount_paid`, `payment_method`, `payment_reference`, `notes`, `created_at`, `updated_at`. +* `selected` applies only when source set includes `advertisers` and means `advertisers.Selected` is truthy. +* `page-in-issue()` accepts exactly one numeric issue ID. It applies only when the source set includes `pages` or `articles`, and means the page row, or the page joined from the article row, belongs to that issue. +* `ad-in-issue()` accepts exactly one numeric issue ID. It applies only when the source set includes one of `advertisers`, `ads`, `pages`, `issues`, or `invoices`, and means the row has an ad in that issue. +* `pending-invoice` applies only when source set includes `invoices` and means `invoices.status = 'pending'`. +* `selected-invoice` applies only when source set includes `invoices` and means the invoice ID is in the runtime selected invoice ID list. +* `invoice-ids(...)` applies only when source set includes `invoices`; it accepts one or more numeric invoice IDs. * `member-or-affiliate-or-parish-council` applies to account data and means: * `accounts.type` is `Member` or `Affiliate`, or * `accounts.name` contains `Parish Council`. @@ -133,10 +179,11 @@ digit = "0"…"9" ; * `field_ref not in (field_ref)` * For `in(field_ref)` / `not in(field_ref)`, the RHS source is treated as a reference source (subquery semantics), not a joined source. * `contains`, `starts-with`, and `ends-with` require a literal right-hand side value (not a field reference). -* In v1.5, mixing built-in sources and custom sources in the same sentence is not supported; validation must fail with a clear error. +* In v1.6, mixing built-in sources and custom sources in the same sentence is not supported; validation must fail with a clear error. * For multi-custom-source sentences, all selected custom sources must be connected by explicit `=` field-to-field predicates (graph-connected join semantics), otherwise validation fails. * `in(field_ref)` / `not in(field_ref)` must compile as subquery membership; the RHS source is a reference source and does not need to appear in `source_expr`. -* Canonical naming in this document uses exact field names as defined in database metadata for the selected source. The implementation must reject invalid names and must not add aliasing/normalization fallback. +* Canonical naming for built-in sources uses the token aliases listed in this document. +* Canonical naming for custom table references uses exact field names as defined in database metadata for the selected source. The implementation must reject invalid custom field names and must not add aliasing/normalization fallback. ### Join-path model (required) @@ -148,6 +195,20 @@ The compiler must use an explicit join graph per source pair. Example v1 join pa * `accounts` -> `renewals`: `accounts.ID = renewals.account_id` * `renewals` -> `contacts`: `renewals.account_id = contacts.Accountid` * `contacts` -> `renewals`: `contacts.Accountid = renewals.account_id` +* `articles` -> `pages`: `articles.PageID = pages.ID` +* `pages` -> `articles`: `pages.ID = articles.PageID` +* `pages` -> `issues`: `pages.Issue = issues.ID` +* `issues` -> `pages`: `issues.ID = pages.Issue` +* `ads` -> `pages`: `ads.PageID = pages.ID` +* `pages` -> `ads`: `pages.ID = ads.PageID` +* `ads` -> `advertisers`: normalized `ads.Advertiser = advertisers.AdvertiserName` +* `advertisers` -> `ads`: normalized `advertisers.AdvertiserName = ads.Advertiser` +* `invoices` -> `ads`: `invoices.ad_id = ads.ID` +* `ads` -> `invoices`: `ads.ID = invoices.ad_id` +* `invoices` -> `issues`: `invoices.issue_id = issues.ID` +* `issues` -> `invoices`: `issues.ID = invoices.issue_id` +* `invoices` -> `pages`: `invoices.issue_id = pages.Issue` +* `pages` -> `invoices`: `pages.Issue = invoices.issue_id` If no approved join path exists between two sources for `and`, parsing/validation must fail with a clear error. @@ -160,7 +221,7 @@ No implicit join behavior is allowed: * `contacts` * `contacts and accounts where fen1-contact` -* `contacts and accounts where fen2-contact` +* `contacts and accounts where primary-contact` * `accounts and contacts where contacts.Accountid = accounts.ID` * `contacts where contacts.Last contains 'smith'` * `contacts and accounts where accounts.Type = 'Member' and contacts.FENContact1 = true` @@ -168,6 +229,11 @@ No implicit join behavior is allowed: * `renewals where pending-renewal` * `renewals where selected-renewal` * `renewals and accounts and contacts where pending-renewal` +* `advertisers where selected` +* `ads and pages where ad-in-issue(202605)` +* `articles and pages where page-in-issue(202605)` +* `invoices where pending-invoice` +* `invoices and ads and advertisers where invoice-ids(101, 102)` * `accounts where accounts.Name not in (members.ExcludedAccounts.ExcludedAccount)` * `members.mailshot_test` @@ -179,6 +245,12 @@ No implicit join behavior is allowed: * `contacts or accounts` (`or` is not supported; use `and` only) * `accounts where pending-renewal` (invalid: filter requires `renewals` source) * `accounts where fen1-contact` (invalid: filter requires `contacts` source) +* `accounts where primary-contact` (invalid: filter requires `contacts` source) +* `ads where selected` (invalid: filter requires `advertisers` source) +* `advertisers where pending-invoice` (invalid: filter requires `invoices` source) +* `pages where ad-in-issue` (invalid: `ad-in-issue` requires exactly one numeric issue ID) +* `ads where page-in-issue(202605)` (invalid: `page-in-issue` requires `pages` or `articles` source) +* `invoices where invoice-ids('abc')` (invalid: `invoice-ids` accepts numeric invoice IDs only) * `renewals where member-or-affiliate-or-parish-council` (invalid: filter requires `accounts` source) * `accounts and contacts where contacts.Accountid contains accounts.ID` (invalid: field-to-field supports only `=`/`!=`) * `accounts where accounts.Name contains members.ExcludedAccounts.ExcludedAccount` (invalid: `contains` requires a literal RHS) @@ -345,8 +417,14 @@ Predefined filter control: * examples: * `Renewal is selected` -> `selected-renewal` * `Renewal is pending` -> `pending-renewal` + * `Advertiser is selected` -> `selected` + * `Page is in issue` -> `page-in-issue(...)` + * `Advertiser has ad in issue` -> `ad-in-issue(...)` + * `Invoice is pending` -> `pending-invoice` + * `Invoice is selected` -> `selected-invoice` + * `Invoice ID is one of` -> `invoice-ids(...)` * `Contact is FEN1` -> `fen1-contact` - * `Contact is FEN2` -> `fen2-contact` + * `Contact is primary` -> `primary-contact` * `Account is member/affiliate/parish council` -> `member-or-affiliate-or-parish-council` Field comparison row: @@ -361,7 +439,7 @@ Field comparison row: Logical composition: -* all rows combine with `and` in v1.5 +* all rows combine with `and` in v1.6 * each row can be negated via checkbox (`not`) * optional group rows allow nested bracketed expressions diff --git a/requirements/renewal_views.md b/requirements/renewal_views.md index 06a8bc3..a76031f 100644 --- a/requirements/renewal_views.md +++ b/requirements/renewal_views.md @@ -31,11 +31,13 @@ Account details: * `accounts.id` * `accounts.name` -* `accounts.account_audience` * `accounts.account_no_of_members` * `accounts.account_type_id` plus joined `picklist_account_type.value` exposed as `accounts.type` * `accounts.public_location_id` plus joined `picklist_public_location.value` exposed as `accounts.public_location` * `accounts.sector_id` plus joined `picklist_sector.value` exposed as `accounts.account_sector` +* `accounts.facebook` +* `accounts.twitter` +* `accounts.other_social` Renewal details: @@ -56,16 +58,16 @@ Contact 1 details (`contacts.is_contact_1 = 1`): * Position/title * Address * Email -* Phone -* Mobile +* Phone 1 +* Phone 2 Contact 2 details (`contacts.is_contact_2 = 1`): * Name * Position/title * Email -* Phone -* Mobile +* Phone 1 +* Phone 2 Fen Edge News contact details (`contacts.is_fen_1 = 1`): @@ -97,7 +99,6 @@ Required columns: * `account_id` -> `accounts.id` * `account_name` -> `accounts.name` -* `account_audience` -> `accounts.account_audience` * `account_no_of_members` -> `accounts.account_no_of_members` * `account_type_id` -> `accounts.account_type_id` * `account_type` -> `picklist_account_type.value` (equivalent to `accounts.type`) @@ -105,6 +106,9 @@ Required columns: * `account_sector` -> `picklist_sector.value` (equivalent to `accounts.account_sector`) * `public_location_id` -> `accounts.public_location_id` * `public_location` -> `picklist_public_location.value` (equivalent to `accounts.public_location`) +* `account_facebook` -> `accounts.facebook` +* `account_twitter` -> `accounts.twitter` +* `account_other_social` -> `accounts.other_social` Filter: @@ -124,7 +128,7 @@ Minimum required prefixed columns: * `..._contact_id` -> `contacts.id` * `..._account_id` -> `contacts.account_id` -* `..._name` -> `TRIM(CONCAT_WS(' ', contacts.first_name, contacts.middle_name, contacts.last_name))` +* `..._name` -> `TRIM(CONCAT_WS(' ', contacts.first_name, contacts.last_name))` * `..._position` -> `contacts.title` * `..._address_1` -> `contacts.contact_address_1` * `..._address_2` -> `contacts.contact_address_2` @@ -132,8 +136,8 @@ Minimum required prefixed columns: * `..._county` -> `contacts.contact_county` * `..._postcode` -> `contacts.contact_postcode` * `..._email` -> `contacts.contact_email_1` -* `..._phone` -> `contacts.home_phone` -* `..._mobile` -> `contacts.mobile` +* `..._phone_1` -> `contacts.phone_1` +* `..._phone_2` -> `contacts.phone_2` ### `renewal_public_contact` @@ -207,15 +211,14 @@ The following names in prior drafts do not match current schema and are explicit * `members.renewals` -> `NON-EXISTENT` in that schema name (use `fenedgec_members.renewals`) * `Account.Name` -> `NON-EXISTENT` (use `accounts.name`) * `Account.Type` -> `NON-EXISTENT` (use `accounts.account_type_id` + `picklist_account_type.slug` for member-status checks, and `picklist_account_type.value` for display) -* `Account.AccountAudience` -> `NON-EXISTENT` (use `accounts.account_audience`) * `Account.AccountNoOfMembers` -> `NON-EXISTENT` (use `accounts.account_no_of_members`) * `Contact.Accountid` -> `NON-EXISTENT` (use `contacts.account_id`) -* `First`/`Middle`/`Last` -> `NON-EXISTENT` (use `first_name`/`middle_name`/`last_name`) +* `First`/`Middle`/`Last` -> `NON-EXISTENT` (use `first_name`/`last_name`) * `JobTitle` -> `NON-EXISTENT`; mapped to `contacts.title` (`AMBIGUOUS: verify business meaning`) * `Address1`/`Address2`/`Town`/`County`/`Postcode` -> `NON-EXISTENT` (use `contact_address_1`/`contact_address_2`/`contact_town`/`contact_county`/`contact_postcode`) * `Email` -> `NON-EXISTENT`; mapped to `contacts.contact_email_1` (`AMBIGUOUS: confirm whether `contact_email_2 ` should also be included`) -* `Home` -> `NON-EXISTENT` (use `contacts.home_phone`) -* `Mobile` -> `NON-EXISTENT` (use `contacts.mobile`) +* `Home` -> `NON-EXISTENT` (use `contacts.phone_1`) +* `Mobile` -> `NON-EXISTENT` (use `contacts.phone_2`) * `Contact1` -> `NON-EXISTENT` (use `contacts.is_contact_1`) * `Contact2` -> `NON-EXISTENT` (use `contacts.is_contact_2`) * `FENContact1` -> `NON-EXISTENT` (use `contacts.is_fen_1`) @@ -231,8 +234,9 @@ The following names in prior drafts do not match current schema and are explicit * `renewal_fen_contact` only includes contacts flagged `is_fen_1 = 1`. * `renewal_public_contact` only includes contacts flagged `is_public_contact = 1`. * `renewal_accounts_with_contacts` returns one row per pending renewal row (no duplicated `renewal_id`). -* `account_audience` and `account_no_of_members` are present in final output. +* `account_no_of_members` is present in final output. * Renewal fields listed above are present in final output. * Account picklist-backed values are present in final output as `account_type`/`account_sector`/`public_location` (or equivalent `accounts.type`/`accounts.account_sector`/`accounts.public_location` aliases). +* Account social fields are present in final output as `account_facebook`, `account_twitter`, and `account_other_social`. * Member-status check is based on `picklist_account_type.slug`, not the label text. * No remote cross-system query is used by these views. diff --git a/scripts/sql/create_renewal_views.sql b/scripts/sql/create_renewal_views.sql index cf45373..8dcad49 100644 --- a/scripts/sql/create_renewal_views.sql +++ b/scripts/sql/create_renewal_views.sql @@ -15,14 +15,16 @@ CREATE VIEW `fenedgec_members`.`renewal_members` AS SELECT a.`id` AS `account_id`, a.`name` AS `account_name`, - a.`account_audience` AS `account_audience`, a.`account_no_of_members` AS `account_no_of_members`, a.`account_type_id` AS `account_type_id`, pat.`value` AS `account_type`, a.`sector_id` AS `sector_id`, ps.`value` AS `account_sector`, a.`public_location_id` AS `public_location_id`, - ppl.`value` AS `public_location` + ppl.`value` AS `public_location`, + a.`facebook` AS `account_facebook`, + a.`twitter` AS `account_twitter`, + a.`other_social` AS `account_other_social` FROM `fenedgec_members`.`accounts` a LEFT JOIN `fenedgec_members`.`picklist_account_type` pat ON pat.`id` = a.`account_type_id` @@ -52,7 +54,7 @@ CREATE VIEW `fenedgec_members`.`renewal_contact_1` AS SELECT c.`id` AS `contact_1_contact_id`, c.`account_id` AS `contact_1_account_id`, - TRIM(CONCAT_WS(' ', c.`first_name`, c.`middle_name`, c.`last_name`)) AS `contact_1_name`, + TRIM(CONCAT_WS(' ', c.`first_name`, c.`last_name`)) AS `contact_1_name`, c.`title` AS `contact_1_position`, c.`contact_address_1` AS `contact_1_address_1`, c.`contact_address_2` AS `contact_1_address_2`, @@ -60,8 +62,8 @@ SELECT c.`contact_county` AS `contact_1_county`, c.`contact_postcode` AS `contact_1_postcode`, c.`contact_email_1` AS `contact_1_email`, - c.`home_phone` AS `contact_1_phone`, - c.`mobile` AS `contact_1_mobile` + c.`phone_1` AS `contact_1_phone_1`, + c.`phone_2` AS `contact_1_phone_2` FROM `fenedgec_members`.`contacts` c INNER JOIN ( SELECT `account_id`, MIN(`id`) AS `min_id` @@ -79,7 +81,7 @@ CREATE VIEW `fenedgec_members`.`renewal_contact_2` AS SELECT c.`id` AS `contact_2_contact_id`, c.`account_id` AS `contact_2_account_id`, - TRIM(CONCAT_WS(' ', c.`first_name`, c.`middle_name`, c.`last_name`)) AS `contact_2_name`, + TRIM(CONCAT_WS(' ', c.`first_name`, c.`last_name`)) AS `contact_2_name`, c.`title` AS `contact_2_position`, c.`contact_address_1` AS `contact_2_address_1`, c.`contact_address_2` AS `contact_2_address_2`, @@ -87,8 +89,8 @@ SELECT c.`contact_county` AS `contact_2_county`, c.`contact_postcode` AS `contact_2_postcode`, c.`contact_email_1` AS `contact_2_email`, - c.`home_phone` AS `contact_2_phone`, - c.`mobile` AS `contact_2_mobile` + c.`phone_1` AS `contact_2_phone_1`, + c.`phone_2` AS `contact_2_phone_2` FROM `fenedgec_members`.`contacts` c INNER JOIN ( SELECT `account_id`, MIN(`id`) AS `min_id` @@ -106,7 +108,7 @@ CREATE VIEW `fenedgec_members`.`renewal_fen_contact` AS SELECT c.`id` AS `fen_contact_contact_id`, c.`account_id` AS `fen_contact_account_id`, - TRIM(CONCAT_WS(' ', c.`first_name`, c.`middle_name`, c.`last_name`)) AS `fen_contact_name`, + TRIM(CONCAT_WS(' ', c.`first_name`, c.`last_name`)) AS `fen_contact_name`, c.`contact_email_1` AS `fen_contact_email` FROM `fenedgec_members`.`contacts` c INNER JOIN ( @@ -125,7 +127,7 @@ CREATE VIEW `fenedgec_members`.`renewal_public_contact` AS SELECT c.`id` AS `public_contact_contact_id`, c.`account_id` AS `public_contact_account_id`, - TRIM(CONCAT_WS(' ', c.`first_name`, c.`middle_name`, c.`last_name`)) AS `public_contact_name`, + TRIM(CONCAT_WS(' ', c.`first_name`, c.`last_name`)) AS `public_contact_name`, c.`public_phone` AS `public_contact_public_phone`, c.`public_email` AS `public_contact_public_email` FROM `fenedgec_members`.`contacts` c @@ -155,8 +157,8 @@ SELECT rc1.`contact_1_county`, rc1.`contact_1_postcode`, rc1.`contact_1_email`, - rc1.`contact_1_phone`, - rc1.`contact_1_mobile`, + rc1.`contact_1_phone_1`, + rc1.`contact_1_phone_2`, rc2.`contact_2_contact_id`, rc2.`contact_2_account_id`, rc2.`contact_2_name`, @@ -167,8 +169,8 @@ SELECT rc2.`contact_2_county`, rc2.`contact_2_postcode`, rc2.`contact_2_email`, - rc2.`contact_2_phone`, - rc2.`contact_2_mobile`, + rc2.`contact_2_phone_1`, + rc2.`contact_2_phone_2`, rcf.`fen_contact_contact_id`, rcf.`fen_contact_account_id`, rcf.`fen_contact_name`,