advertising fixups

docs/todo.md

@@ -0,0 +1,3 @@
+Split advertiser name into forename and surname
+
+Add ad sizes into mailmerge

feca_mailshots_plugin/feca_mailshots_plugin.php

@@ -3,7 +3,7 @@
  * Plugin Name: FECA Mailshots
  * Plugin URI: https://fenedge.co.uk/
  * Description: FECA mailshots plugin.
- * Version: 1.1.0
+ * Version: 1.1.5
  * Requires at least: 6.0
  * Requires PHP: 7.4
  * Author: FECA

feca_mailshots_plugin/src/Admin/DataSourcesAdminPage.php

@@ -184,16 +184,11 @@ final class DataSourcesAdminPage
         echo '<div class="feca-builder-grid">';
         echo '<div class="feca-builder-col-left">';
         echo '<h3 class="feca-modal-title">Data Sources</h3>';
-        echo '<label><input type="checkbox" class="ds-source-built" value="contacts"> contacts</label><br>';
+        foreach (array_keys($sourceFieldsMap) as $builtInSource) {
-        echo '<label><input type="checkbox" class="ds-source-built" value="accounts"> accounts</label><br>';
+            $escapedSource = htmlspecialchars((string) $builtInSource, ENT_QUOTES);
-        echo '<label><input type="checkbox" class="ds-source-built" value="renewals"> renewals</label><br>';
+            echo '<label><input type="checkbox" class="ds-source-built" value="' . $escapedSource . '"> ' . $escapedSource . '</label><br>';
-        echo '<label><input type="checkbox" class="ds-source-built" value="grants"> grants</label><br>';
+        }
-        echo '<label><input type="checkbox" class="ds-source-built" value="advertisers"> advertisers</label><br>';
+        echo '<br>';
-        echo '<label><input type="checkbox" class="ds-source-built" value="ads"> ads</label><br>';
-        echo '<label><input type="checkbox" class="ds-source-built" value="pages"> pages</label><br>';
-        echo '<label><input type="checkbox" class="ds-source-built" value="articles"> articles</label><br>';
-        echo '<label><input type="checkbox" class="ds-source-built" value="issues"> issues</label><br>';
-        echo '<label><input type="checkbox" class="ds-source-built" value="invoices"> invoices</label><br><br>';
         echo '<strong>Add custom source</strong><br>';
         echo '<label>Schema <select id="ds-builder-schema"><option value="">Select schema</option>';
         foreach ($schemaList as $schema) {

feca_mailshots_plugin/src/Application/DslCompiler.php

@@ -113,7 +113,7 @@ final class DslCompiler
                 $value = '%' . $value;
             }
             $params[] = $value;
-            $sql = sprintf('%s %s ?', $lhs, $op);
+            $sql = sprintf('%s %s ?', $this->nullAsEmptySql($lhs), $op);
             return $predicate['not'] ? 'NOT (' . $sql . ')' : $sql;
         }
 
@@ -126,7 +126,12 @@ final class DslCompiler
         if ($predicate['type'] === 'comparison_field') {
             $lhs = $this->fieldRefSql($predicate['lhs']);
             $rhs = $this->fieldRefSql($predicate['rhs']);
-            $sql = sprintf('%s %s %s', $lhs, $predicate['op'], $rhs);
+            $sql = sprintf(
+                '%s %s %s',
+                $this->nullAsEmptySql($lhs),
+                $predicate['op'],
+                $this->nullAsEmptySql($rhs)
+            );
             return $predicate['not'] ? 'NOT (' . $sql . ')' : $sql;
         }
 
@@ -161,6 +166,11 @@ final class DslCompiler
         return '(' . $fieldSql . ' IS NULL OR ' . $fieldSql . " = '')";
     }
 
+    private function nullAsEmptySql(string $fieldSql): string
+    {
+        return 'COALESCE(' . $fieldSql . ", '')";
+    }
+
     /** @param list<mixed> $args @param list<mixed> &$params @param list<string> $sources */
     private function compileFilter(string $name, array $args, array &$params, array $sources): string
     {
@@ -213,8 +223,8 @@ final class DslCompiler
     /** @return array{string,string} */
     private function rewriteJoinRef(string $left, string $right): array
     {
-        $leftParts = explode('.', $left, 2);
+        $leftParts = $this->splitJoinRef($left);
-        $rightParts = explode('.', $right, 2);
+        $rightParts = $this->splitJoinRef($right);
 
         return [
             $this->alias($leftParts[0]) . '.`' . str_replace('`', '``', $leftParts[1]) . '`',
@@ -222,6 +232,17 @@ final class DslCompiler
         ];
     }
 
+    /** @return array{string,string} */
+    private function splitJoinRef(string $reference): array
+    {
+        $separator = strrpos($reference, '.');
+        if ($separator === false || $separator === 0 || $separator === strlen($reference) - 1) {
+            throw new AppError('dsl_compile', 'Invalid join field reference', ['reference' => $reference]);
+        }
+
+        return [substr($reference, 0, $separator), substr($reference, $separator + 1)];
+    }
+
     /** @param array{source:string,field:string} $fieldRef */
     private function fieldRefSql(array $fieldRef): string
     {

feca_mailshots_plugin/src/Application/DslValidator.php

@@ -52,10 +52,10 @@ final class DslValidator
                 $errors[] = 'Unknown source: ' . $source;
                 continue;
             }
-            if (str_contains($source, '.')) {
+            if ($this->metadata->isBuiltInSource($source)) {
-                $hasCustom = true;
-            } else {
                 $hasBuiltIn = true;
+            } else {
+                $hasCustom = true;
             }
         }
 

feca_mailshots_plugin/src/Application/SourceMetadataProvider.php

@@ -8,6 +8,8 @@ interface SourceMetadataProvider
 {
     public function sourceExists(string $source): bool;
 
+    public function isBuiltInSource(string $source): bool;
+
     /** @return list<string> */
     public function sourceFields(string $source): array;
 

feca_mailshots_plugin/src/Domain/DslParser.php

@@ -156,19 +156,23 @@ final class DslParser
         if ($this->match('.')) {
             $ident .= '.' . $field;
             $field = $this->expect('IDENT')->value;
+        } else {
+            $ident = strtolower($ident);
         }
-        return ['source' => strtolower($ident), 'field' => $field];
+        return ['source' => $ident, 'field' => $field];
     }
 
     /** @return array{source:string, field:string} */
     private function parseFieldRef(): array
     {
-        $src = strtolower($this->expect('IDENT')->value);
+        $src = $this->expect('IDENT')->value;
         $this->expect('.');
         $field = $this->expect('IDENT')->value;
         if ($this->match('.')) {
-            $src .= '.' . strtolower($field);
+            $src .= '.' . $field;
             $field = $this->expect('IDENT')->value;
+        } else {
+            $src = strtolower($src);
         }
         return ['source' => $src, 'field' => $field];
     }

feca_mailshots_plugin/src/Infrastructure/BasicSmtpSender.php

@@ -54,7 +54,7 @@ final class BasicSmtpSender implements SmtpSender
             $this->cmd($fp, 'DATA', [354]);
 
             $raw = $this->buildMime($from, $fromName, $to, $cc, $bcc, $subject, $htmlBody, $replyTo, $attachments);
-            fwrite($fp, $raw);
+            fwrite($fp, $this->dotStuff($raw));
             fwrite($fp, "\r\n.\r\n");
             $this->expect($fp, [250]);
             $this->cmd($fp, 'QUIT', [221], false);
@@ -85,15 +85,16 @@ final class BasicSmtpSender implements SmtpSender
         $headers[] = 'MIME-Version: 1.0';
         if ($attachments === []) {
             $headers[] = 'Content-Type: text/html; charset=UTF-8';
-            return implode("\r\n", $headers) . "\r\n\r\n" . $htmlBody;
+            $headers[] = 'Content-Transfer-Encoding: quoted-printable';
+            return implode("\r\n", $headers) . "\r\n\r\n" . $this->encodeQuotedPrintable($htmlBody);
         }
 
         $boundary = 'feca_mailshots_' . bin2hex(random_bytes(12));
         $mime = implode("\r\n", array_merge($headers, ['Content-Type: multipart/mixed; boundary="' . $boundary . '"'])) . "\r\n\r\n";
         $mime .= '--' . $boundary . "\r\n";
         $mime .= 'Content-Type: text/html; charset=UTF-8' . "\r\n";
-        $mime .= 'Content-Transfer-Encoding: 8bit' . "\r\n\r\n";
+        $mime .= 'Content-Transfer-Encoding: quoted-printable' . "\r\n\r\n";
-        $mime .= $htmlBody . "\r\n";
+        $mime .= $this->encodeQuotedPrintable($htmlBody) . "\r\n";
 
         foreach ($attachments as $attachment) {
             $filename = trim((string) ($attachment['filename'] ?? 'attachment.bin'));
@@ -180,6 +181,18 @@ final class BasicSmtpSender implements SmtpSender
         }
     }
 
+    private function encodeQuotedPrintable(string $body): string
+    {
+        $body = preg_replace("/\r\n|\r|\n/", "\r\n", $body) ?? $body;
+        return quoted_printable_encode($body);
+    }
+
+    private function dotStuff(string $data): string
+    {
+        $data = preg_replace("/\r\n|\r|\n/", "\r\n", $data) ?? $data;
+        return preg_replace('/(?m)^\./', '..', $data) ?? $data;
+    }
+
     /** @param list<int> $codes */
     private function cmd($fp, string $cmd, array $codes, bool $throwOnMismatch = true): ?string
     {

feca_mailshots_plugin/src/Infrastructure/DatabaseSourceMetadataProvider.php

@@ -24,7 +24,8 @@ final class DatabaseSourceMetadataProvider implements SourceMetadataProvider
     {
         $this->router = $router;
         $fen = $router->fenDbName();
-        $this->builtInSources = ['contacts', 'accounts', 'renewals', 'grants', 'advertisers', 'ads', 'pages', 'articles', 'issues', 'invoices'];
+        $adSizes = 'ad_sizes';
+        $this->builtInSources = ['contacts', 'accounts', 'renewals', 'grants', 'advertisers', 'ads', 'pages', 'articles', 'issues', 'invoices', $adSizes];
 
         $this->builtInTables = [
             'contacts' => 'contacts',
@@ -37,6 +38,7 @@ final class DatabaseSourceMetadataProvider implements SourceMetadataProvider
             'articles' => $fen . '.Articles',
             'issues' => $fen . '.Issues',
             'invoices' => $fen . '.invoices',
+            $adSizes => $fen . '.Ad_Sizes',
         ];
 
         $this->joinMap = [
@@ -62,6 +64,8 @@ final class DatabaseSourceMetadataProvider implements SourceMetadataProvider
             'issues|invoices' => ['left' => 'issues.ID', 'right' => 'invoices.issue_id'],
             'invoices|pages' => ['left' => 'invoices.issue_id', 'right' => 'pages.Issue'],
             'pages|invoices' => ['left' => 'pages.Issue', 'right' => 'invoices.issue_id'],
+            'ads|' . $adSizes => ['left' => 'ads.AdSize', 'right' => $adSizes . '.SizeName'],
+            $adSizes . '|ads' => ['left' => $adSizes . '.SizeName', 'right' => 'ads.AdSize'],
         ];
     }
 
@@ -86,6 +90,11 @@ final class DatabaseSourceMetadataProvider implements SourceMetadataProvider
         return false;
     }
 
+    public function isBuiltInSource(string $source): bool
+    {
+        return in_array($source, $this->builtInSources, true);
+    }
+
     public function sourceFields(string $source): array
     {
         if (in_array($source, $this->builtInSources, true)) {

requirements/mailshot_data_source.md

@@ -134,6 +134,7 @@ digit              = "0"…"9" ;
 * For custom sources, an explicit field equality predicate can provide join semantics.
 * `where` applies after source composition.
 * `not` negates only the next predicate/group.
+* Comparison operators (`=`, `!=`, `contains`, `starts-with`, and `ends-with`) treat a database `NULL` field value as an empty string. This makes a negated comparison the complete logical inverse of its positive form; for example, `not (source.field contains 'text')` includes rows where `source.field` is `NULL`.
 * Predefined predicates are `selected-renewal`, `pending-renewal`, `selected`, `issue`, `pending-invoice`, `selected-invoice`, `invoice-ids`, `fen1-contact`, `primary-contact`, `member-or-affiliate-or-parish-council`, and `account-has-article-in-issue`.
 * `renewals` is a built-in source mapped to membership renewal rows.
 * `pending-renewal` applies only when source set includes `renewals` and means `renewals.status = 'pending'`.
@@ -148,6 +149,7 @@ digit              = "0"…"9" ;
   * `articles` maps to `fen.Articles`.
   * `issues` maps to `fen.Issues`.
   * `invoices` maps to `fen.invoices`.
+  * `ad_sizes` maps to `${FEN_REMOTE_MYSQL_DB}.Ad_Sizes`. The configured database/schema name is an implementation detail and must not appear in the source name shown by the builder or in generated DSL.
 * Required FEN source fields and canonical token aliases:
   * `advertisers`: `name` / `advertisername` from `AdvertiserName`.
   * `ads`: `id` from `ID`, `advertiser` from `Advertiser`, `adsize` / `size` from `AdSize`, `price` from `Price`, `issue` from the related page `Issue`, `pageid` from `PageID`, `state` from `State`, `notes` from `Notes`.
@@ -162,6 +164,7 @@ digit              = "0"…"9" ;
   * `fen.ads`: `ID`, `PageID`, `AdSize`, `Advertiser`, `Price`, `State`, `Notes`.
   * `fen.advertisers`: `Entry ID`, `AdvertiserName`, `title`, `contact_name`, `address_1`, `address_2`, `town`, `post_code`, `Description`, `IsLapsed?`, `Home Phone`, `Phone`, `Email`, `Selected`.
   * `fen.invoices`: `id`, `issue_id`, `ad_id`, `invoice_number`, `invoice_date`, `due_date`, `invoice_page`, `invoice_size`, `invoice_price`, `status`, `payment_date`, `amount_paid`, `payment_method`, `payment_reference`, `notes`, `created_at`, `updated_at`.
+  * `fen.Ad_Sizes`: `SizeName`; all readable table fields are exposed using their physical names.
 * `selected` applies only when source set includes `advertisers` and means `advertisers.Selected` is truthy.
 * `issue(<issue>)` accepts exactly one numeric issue ID. It applies when the source set includes one of `advertisers`, `ads`, `pages`, `articles`, `issues`, or `invoices`, and means the row is associated with that issue. The compiler may traverse hidden approved paths to apply the filter, but only explicitly cited sources contribute fields to the result/template context.
 * When `issues` is explicitly appended to an otherwise joined source set and a non-negated `issue(<issue>)` filter is present, `issues` may be attached as a one-row issue context source even when there is no direct approved join path from the preceding source. Without that constraining issue filter, the join must remain invalid.
@@ -207,6 +210,8 @@ The compiler must use an explicit join graph per source pair. Example v1 join pa
 * `advertisers` -> `ads`: normalized `advertisers.AdvertiserName = ads.Advertiser`
 * `invoices` -> `ads`: `invoices.ad_id = ads.ID`
 * `ads` -> `invoices`: `ads.ID = invoices.ad_id`
+* `ads` -> `ad_sizes`: `ads.AdSize = ad_sizes.SizeName`
+* `ad_sizes` -> `ads`: `ad_sizes.SizeName = ads.AdSize`
 * `invoices` -> `issues`: `invoices.issue_id = issues.ID`
 * `issues` -> `invoices`: `issues.ID = invoices.issue_id`
 * `invoices` -> `pages`: `invoices.issue_id = pages.Issue`

results/manual-testing-1.md

@@ -26,6 +26,6 @@
 * [X] Mailshot data sources.  Add "Duplicate" button per row.
 * [X] Select a single row in "review recipients".   "Run Mailshot to selected rows" says "Run mailshot to 1 selected rows?".  Yes -> error "None of the selected recipient rows exist in the current query result."
 * [X] Error report from wordpress. - packaging was missing a file
-* [ ] Subject to "Memory box café generates an error from receiveing yahoo email "subject contains an invalid character".   Need to check rules for character set in subject and modify special characters accordingly.  Problem characters include quote (All Saint's) and e accent (café).
+* [X] Subject to "Memory box café generates an error from receiveing yahoo email "subject contains an invalid character".   Need to check rules for character set in subject and modify special characters accordingly.  Problem characters include quote (All Saint's) and e accent (café).
-* [ ] (patch 32, 33) Test email including these characters critical errors wordpress.  Perhaps because "Last Run Rows" was non-empty and related to a previous different mailshot / data source.   Still critical errors after attempt to make more robust.
+* [X] (patch 32, 33) Test email including these characters critical errors wordpress.  Perhaps because "Last Run Rows" was non-empty and related to a previous different mailshot / data source.   Still critical errors after attempt to make more robust.
-* [ ] patch 30 also criticals sending to "test" mailshot,  but OK sending to fen contacts.
+* [X] patch 30 also criticals sending to "test" mailshot,  but OK sending to fen contacts.

tests/unit/FakeMetadataProvider.php

@@ -10,11 +10,17 @@ final class FakeMetadataProvider implements SourceMetadataProvider
 {
     /** @var array<string, list<string>> */
     private array $fields;
+    /** @var list<string> */
+    private array $builtInSources;
 
-    /** @param array<string, list<string>> $fields */
+    /** @param array<string, list<string>> $fields @param list<string> $additionalBuiltInSources */
-    public function __construct(array $fields)
+    public function __construct(array $fields, array $additionalBuiltInSources = [])
     {
         $this->fields = $fields;
+        $this->builtInSources = array_values(array_unique(array_merge(
+            array_values(array_filter(array_keys($fields), static fn(string $source): bool => !str_contains($source, '.'))),
+            $additionalBuiltInSources
+        )));
     }
 
     public function sourceExists(string $source): bool
@@ -22,6 +28,11 @@ final class FakeMetadataProvider implements SourceMetadataProvider
         return isset($this->fields[$source]);
     }
 
+    public function isBuiltInSource(string $source): bool
+    {
+        return in_array($source, $this->builtInSources, true);
+    }
+
     public function sourceFields(string $source): array
     {
         return $this->fields[$source] ?? [];
@@ -56,6 +67,10 @@ final class FakeMetadataProvider implements SourceMetadataProvider
             'invoices|pages' => ['left' => 'invoices.issue_id', 'right' => 'pages.Issue'],
             'pages|invoices' => ['left' => 'pages.Issue', 'right' => 'invoices.issue_id'],
         ];
+        if (isset($this->fields['ad_sizes'])) {
+            $pairs['ads|ad_sizes'] = ['left' => 'ads.AdSize', 'right' => 'ad_sizes.SizeName'];
+            $pairs['ad_sizes|ads'] = ['left' => 'ad_sizes.SizeName', 'right' => 'ads.AdSize'];
+        }
         return $pairs[$left . '|' . $right] ?? null;
     }
 

tests/unit/test_basic_smtp_sender_mime.php

@@ -40,6 +40,35 @@ if (strpos($plain, 'From: =?UTF-8?B?RnJvbSBOYW1l?= <from@example.org>') === fals
     fwrite(STDERR, "From display name missing expected RFC 2047 encoding\n");
     exit(1);
 }
+if (strpos($plain, 'Content-Transfer-Encoding: quoted-printable') === false) {
+    fwrite(STDERR, "Plain MIME rendering should use quoted-printable HTML body encoding\n");
+    exit(1);
+}
+
+$longHtml = '<p>' . str_repeat('LongHtmlSegment', 120) . '</p>';
+$longPlain = $buildMime->invoke(
+    $sender,
+    'from@example.org',
+    'From Name',
+    ['to@example.org'],
+    [],
+    [],
+    'Subject',
+    $longHtml,
+    null,
+    []
+);
+
+if (!is_string($longPlain)) {
+    fwrite(STDERR, "Long plain MIME rendering did not return a string\n");
+    exit(1);
+}
+foreach (preg_split('/\r\n/', $longPlain) ?: [] as $line) {
+    if (strlen($line) > 998) {
+        fwrite(STDERR, "Plain MIME contains an RFC 5322 overlong line\n");
+        exit(1);
+    }
+}
 
 $bytes = random_bytes(256 * 1024);
 $mime = $buildMime->invoke(
@@ -67,9 +96,19 @@ if (strpos($mime, 'Content-Type: image/jpeg; name="sample.jpg"') === false) {
     fwrite(STDERR, "Attachment MIME part missing expected content-type/filename\n");
     exit(1);
 }
+if (strpos($mime, 'Content-Transfer-Encoding: quoted-printable') === false) {
+    fwrite(STDERR, "Multipart HTML part should use quoted-printable body encoding\n");
+    exit(1);
+}
 if (strpos($mime, base64_encode(substr($bytes, 0, 24))) === false) {
     fwrite(STDERR, "Attachment bytes do not appear to be base64-encoded into MIME payload\n");
     exit(1);
 }
+foreach (preg_split('/\r\n/', $mime) ?: [] as $line) {
+    if (strlen($line) > 998) {
+        fwrite(STDERR, "Multipart MIME contains an RFC 5322 overlong line\n");
+        exit(1);
+    }
+}
 
 echo "BasicSmtpSender MIME regression test passed\n";

tests/unit/test_dsl_accounts_picklists.php

@@ -29,7 +29,7 @@ if (strpos($sql, 's_accounts.`account_type_id` AS `accounts.account_type_id`') =
     fwrite(STDERR, "Expected physical accounts.account_type_id projection\n");
     exit(1);
 }
-if (strpos($sql, 's_accounts.`account_type_id` = ?') === false) {
+if (strpos($sql, "COALESCE(s_accounts.`account_type_id`, '') = ?") === false) {
     fwrite(STDERR, "Expected predicate accounts.account_type_id to compile against physical column\n");
     exit(1);
 }

tests/unit/test_dsl_fen_builtins.php

@@ -10,6 +10,7 @@ use FecaMailshots\Application\DslValidator;
 use FecaMailshots\Domain\DslParser;
 use FecaMailshots\Tests\Unit\FakeMetadataProvider;
 
+$adSizesSource = 'ad_sizes';
 $metadata = new FakeMetadataProvider([
     'advertisers' => ['AdvertiserName', 'Selected', 'IsLapsed?'],
     'ads' => ['ID', 'Advertiser', 'AdSize', 'Price', 'PageID', 'State', 'Notes'],
@@ -17,6 +18,7 @@ $metadata = new FakeMetadataProvider([
     'articles' => ['ID', 'PageID', 'ArticleNumber', 'Content', 'MemberName', 'Author', 'DCN', 'ArticleWords', 'OtherWords', 'OtherContent'],
     'issues' => ['ID', 'IssueMonths', 'Description'],
     'invoices' => ['id', 'issue', 'issue_id', 'ad_id', 'invoice_number', 'status'],
+    $adSizesSource => ['SizeName', 'Width', 'Height'],
     'contacts' => ['id', 'account_id', 'last_name', 'contact_email_1', 'is_fen_1', 'is_deleted'],
     'accounts' => ['id', 'name', 'is_deleted'],
 ]);
@@ -90,6 +92,37 @@ foreach ([
     }
 }
 
+$adSizesDsl = 'ads and ad_sizes where issue(125)';
+$adSizesAst = $parser->parse($adSizesDsl);
+$adSizesValidation = $validator->validate($adSizesAst);
+if (($adSizesValidation['errors'] ?? []) !== []) {
+    fwrite(STDERR, "Expected valid DSL {$adSizesDsl}: " . json_encode($adSizesValidation['errors']) . "\n");
+    exit(1);
+}
+$adSizesSql = (string) ($compiler->compile($adSizesAst)['sql'] ?? '');
+foreach ([
+    'INNER JOIN `ad_sizes` AS s_ad_sizes ON s_ads.`AdSize` = s_ad_sizes.`SizeName`',
+    's_ad_sizes.`Width` AS `ad_sizes.Width`',
+] as $needle) {
+    if (strpos($adSizesSql, $needle) === false) {
+        fwrite(STDERR, "Expected Ad_Sizes SQL to contain {$needle}\n{$adSizesSql}\n");
+        exit(1);
+    }
+}
+
+$reverseAdSizesDsl = 'ad_sizes and ads where ad_sizes.SizeName = ads.AdSize';
+$reverseAdSizesAst = $parser->parse($reverseAdSizesDsl);
+$reverseAdSizesValidation = $validator->validate($reverseAdSizesAst);
+if (($reverseAdSizesValidation['errors'] ?? []) !== []) {
+    fwrite(STDERR, "Expected valid reverse DSL {$reverseAdSizesDsl}: " . json_encode($reverseAdSizesValidation['errors']) . "\n");
+    exit(1);
+}
+$reverseAdSizesSql = (string) ($compiler->compile($reverseAdSizesAst)['sql'] ?? '');
+if (strpos($reverseAdSizesSql, 'ON s_ad_sizes.`SizeName` = s_ads.`AdSize`') === false) {
+    fwrite(STDERR, "Expected reverse Ad_Sizes implied join\n{$reverseAdSizesSql}\n");
+    exit(1);
+}
+
 $quotedFieldDsl = 'advertisers where advertisers.`IsLapsed?` = true';
 $quotedFieldAst = $parser->parse($quotedFieldDsl);
 $quotedFieldValidation = $validator->validate($quotedFieldAst);
@@ -98,11 +131,25 @@ if (($quotedFieldValidation['errors'] ?? []) !== []) {
     exit(1);
 }
 $quotedFieldSql = (string) ($compiler->compile($quotedFieldAst)['sql'] ?? '');
-if (strpos($quotedFieldSql, 's_advertisers.`IsLapsed?` = ?') === false) {
+if (strpos($quotedFieldSql, "COALESCE(s_advertisers.`IsLapsed?`, '') = ?") === false) {
     fwrite(STDERR, "Expected quoted physical field to compile directly\n{$quotedFieldSql}\n");
     exit(1);
 }
 
+$notContainsDsl = "advertisers where not (advertisers.AdvertiserName contains 'discount')";
+$notContainsSql = (string) ($compiler->compile($parser->parse($notContainsDsl))['sql'] ?? '');
+if (strpos($notContainsSql, "NOT (COALESCE(s_advertisers.`AdvertiserName`, '') LIKE ?)") === false) {
+    fwrite(STDERR, "Expected negated contains to include NULL values via empty-string normalization\n{$notContainsSql}\n");
+    exit(1);
+}
+
+$fieldComparisonDsl = 'contacts and accounts where contacts.account_id = accounts.id';
+$fieldComparisonSql = (string) ($compiler->compile($parser->parse($fieldComparisonDsl))['sql'] ?? '');
+if (strpos($fieldComparisonSql, "COALESCE(s_contacts.`account_id`, '') = COALESCE(s_accounts.`id`, '')") === false) {
+    fwrite(STDERR, "Expected field comparison to normalize NULL values to empty strings\n{$fieldComparisonSql}\n");
+    exit(1);
+}
+
 $issueCases = [
     'issues where issue(202605)' => 's_issues.`ID` = ?',
     'articles where issue(202605)' => 'p_article_issue.`Issue` = ?',