Snapshot: MLS sync fixes, image refresh, plugin/theme updates
MLS plugin fixes from this session: - Fix silent insert failures: location column NOT NULL was rejecting wpdb->insert calls, causing ~18k new properties since Dec 2025 to be lost. Inserts now build raw SQL with ST_PointFromText so the spatial column is populated atomically. - Auto-refresh expired media URLs in MLS_Media_Handler::fetch_and_cache(), guarded by a property-level GET_LOCK so concurrent fetches share one API refresh. - Normalize WP_Error to null in mls_get_property_image() so callers can rely on the documented string|null contract. - Support comma-separated property_type filters in MLS_Query and MLS_Cluster so the homepage "View All Commercial" link (?property_type=Commercial+Sale,Land,Farm) actually filters correctly. - Incremental sync now looks back 10 minutes past the latest modification timestamp as a safety margin against missed records. - Smart sync exits silently (info-level, not warning) when a full sync is in progress. Operational: - New cron: weekly full sync Sundays at 3 AM (/usr/local/bin/mls-full-sync). - New cron: hourly 2GB cap on mls-thumbnails/ and cache/transformed-images/ (/usr/local/bin/mls-image-cache-cap). - Logrotate config for wp-content/debug.log (2-day retention, daily rotation, delaycompress). Repo policy: - CLAUDE.md updated with explicit "commit everything except build artifacts" policy. - .gitignore: untrack runtime image caches and debug.log rotations. Other modifications in this snapshot are pre-existing in-flight theme/plugin/db_content_updates work. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -21,6 +21,7 @@ class MLS_Sync_Engine {
|
||||
const TYPE_FULL = 'full';
|
||||
const TYPE_INCREMENTAL = 'incremental';
|
||||
const TYPE_MEDIA = 'media';
|
||||
const TYPE_MEDIA_REFRESH = 'media_refresh';
|
||||
|
||||
/**
|
||||
* Sync statuses
|
||||
@@ -65,6 +66,8 @@ class MLS_Sync_Engine {
|
||||
'updated' => 0,
|
||||
'deleted' => 0,
|
||||
'errors' => 0,
|
||||
'homeproz_media_downloaded' => 0,
|
||||
'homeproz_media_skipped' => 0,
|
||||
);
|
||||
|
||||
/**
|
||||
@@ -387,6 +390,12 @@ class MLS_Sync_Engine {
|
||||
|
||||
$this->logger->info('Incremental sync completed', $this->stats);
|
||||
|
||||
// Download pending media for HomeProz properties
|
||||
// This catches any HomeProz listings that have media records but images weren't downloaded
|
||||
$media_stats = $this->download_pending_homeproz_media($dry_run);
|
||||
$this->stats['homeproz_media_downloaded'] = $media_stats['downloaded'];
|
||||
$this->stats['homeproz_media_skipped'] = $media_stats['skipped'];
|
||||
|
||||
} catch (Exception $e) {
|
||||
$this->logger->error('Incremental sync failed', array('error' => $e->getMessage()));
|
||||
|
||||
@@ -410,6 +419,211 @@ class MLS_Sync_Engine {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run media refresh sync for properties with expiring media URLs
|
||||
*
|
||||
* Fetches fresh data for properties whose media URLs will expire within
|
||||
* the specified number of days. This prevents on-demand API calls when
|
||||
* visitors try to view images with expired URLs.
|
||||
*
|
||||
* If a property is no longer listed (not Active/Pending or MlgCanView=false),
|
||||
* it will be removed from the local database.
|
||||
*
|
||||
* @param int $days_ahead Number of days to look ahead for expiring media (default: 3)
|
||||
* @param bool $dry_run If true, don't make changes
|
||||
* @param callable|null $progress_callback Callback for progress updates
|
||||
* @return array Sync results
|
||||
*/
|
||||
public function run_media_refresh_sync($days_ahead = 3, $dry_run = false, $progress_callback = null) {
|
||||
global $wpdb;
|
||||
|
||||
$this->logger->info('Starting media refresh sync', array(
|
||||
'days_ahead' => $days_ahead,
|
||||
'dry_run' => $dry_run,
|
||||
));
|
||||
|
||||
$this->progress_callback = $progress_callback;
|
||||
|
||||
if (!$dry_run) {
|
||||
$this->sync_state_id = $this->create_sync_state(self::TYPE_MEDIA_REFRESH);
|
||||
$this->logger->set_sync_state($this->sync_state_id);
|
||||
}
|
||||
|
||||
$this->stats = array(
|
||||
'processed' => 0,
|
||||
'created' => 0,
|
||||
'updated' => 0,
|
||||
'deleted' => 0,
|
||||
'errors' => 0,
|
||||
);
|
||||
|
||||
try {
|
||||
// Find properties with media expiring within X days
|
||||
$expiry_threshold = gmdate('Y-m-d H:i:s', strtotime("+{$days_ahead} days"));
|
||||
|
||||
// Include Active/Pending properties, plus Closed HomeProz properties
|
||||
// (HomeProz wants to keep sold property images for portfolio)
|
||||
$properties = $wpdb->get_results($wpdb->prepare(
|
||||
"SELECT listing_key, listing_id, media_expires_at
|
||||
FROM {$this->db->properties_table()}
|
||||
WHERE (media_expires_at IS NULL OR media_expires_at <= %s)
|
||||
AND (standard_status IN ('Active', 'Pending') OR (standard_status = 'Closed' AND is_homeproz = 1))
|
||||
ORDER BY media_expires_at ASC",
|
||||
$expiry_threshold
|
||||
));
|
||||
|
||||
$total = count($properties);
|
||||
$this->logger->info("Found {$total} properties with expiring media");
|
||||
|
||||
$this->emit_progress('media_refresh_start', array(
|
||||
'total' => $total,
|
||||
'expiry_threshold' => $expiry_threshold,
|
||||
));
|
||||
|
||||
// Process in batches of 25 (MLS Grid max with $expand)
|
||||
$batch_size = 25;
|
||||
$batches = array_chunk($properties, $batch_size);
|
||||
$batch_num = 0;
|
||||
|
||||
foreach ($batches as $batch) {
|
||||
$batch_num++;
|
||||
|
||||
// Build array of listing_ids for this batch
|
||||
$listing_ids = array_map(function($prop) {
|
||||
return $prop->listing_id;
|
||||
}, $batch);
|
||||
|
||||
// Fetch batch from API
|
||||
$start_time = microtime(true);
|
||||
$this->emit_progress('api_request', array(
|
||||
'method' => 'GET',
|
||||
'url' => 'Property',
|
||||
'params' => array('batch' => $batch_num, 'count' => count($listing_ids)),
|
||||
));
|
||||
|
||||
$response = $this->api_client->get_properties_by_ids($listing_ids);
|
||||
$elapsed = round((microtime(true) - $start_time) * 1000);
|
||||
|
||||
if (is_wp_error($response)) {
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => false,
|
||||
'status_code' => 0,
|
||||
'error' => $response->get_error_message(),
|
||||
'response_time' => $elapsed,
|
||||
));
|
||||
|
||||
// Mark all in batch as errors
|
||||
foreach ($batch as $prop) {
|
||||
$this->stats['processed']++;
|
||||
$this->stats['errors']++;
|
||||
}
|
||||
$this->logger->warning('Failed to fetch batch for media refresh', array(
|
||||
'batch' => $batch_num,
|
||||
'error' => $response->get_error_message(),
|
||||
));
|
||||
continue;
|
||||
}
|
||||
|
||||
$returned_count = isset($response['value']) ? count($response['value']) : 0;
|
||||
$this->emit_progress('api_response', array(
|
||||
'success' => true,
|
||||
'status_code' => 200,
|
||||
'response_time' => $elapsed,
|
||||
'record_count' => $returned_count,
|
||||
));
|
||||
|
||||
// Index returned properties by ListingId
|
||||
$returned_by_id = array();
|
||||
if (isset($response['value'])) {
|
||||
foreach ($response['value'] as $property_data) {
|
||||
if (isset($property_data['ListingId'])) {
|
||||
$returned_by_id[$property_data['ListingId']] = $property_data;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process each property in the batch
|
||||
foreach ($batch as $prop) {
|
||||
$this->stats['processed']++;
|
||||
|
||||
if (isset($returned_by_id[$prop->listing_id])) {
|
||||
// Property found - process_property handles its own progress events
|
||||
$property_data = $returned_by_id[$prop->listing_id];
|
||||
|
||||
if (!$dry_run) {
|
||||
$this->process_property($property_data, false);
|
||||
} else {
|
||||
$this->stats['updated']++;
|
||||
$this->emit_progress('property_skipped', array(
|
||||
'listing_key' => $prop->listing_key,
|
||||
));
|
||||
}
|
||||
} else {
|
||||
// Property not in API response - may have been removed
|
||||
if (!$dry_run) {
|
||||
$this->delete_property($prop->listing_key);
|
||||
}
|
||||
$this->stats['deleted']++;
|
||||
$this->emit_progress('property_deleted', array(
|
||||
'listing_key' => $prop->listing_key,
|
||||
'reason' => 'Not found in API',
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Update sync state after each batch
|
||||
if (!$dry_run) {
|
||||
$this->update_sync_state(array(
|
||||
'records_processed' => $this->stats['processed'],
|
||||
'records_updated' => $this->stats['updated'],
|
||||
'records_deleted' => $this->stats['deleted'],
|
||||
));
|
||||
}
|
||||
|
||||
// Emit batch/page complete
|
||||
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
|
||||
}
|
||||
|
||||
// Mark sync as completed
|
||||
if (!$dry_run) {
|
||||
$this->update_sync_state(array(
|
||||
'status' => self::STATUS_COMPLETED,
|
||||
'completed_at' => current_time('mysql'),
|
||||
'records_processed' => $this->stats['processed'],
|
||||
'records_updated' => $this->stats['updated'],
|
||||
'records_deleted' => $this->stats['deleted'],
|
||||
));
|
||||
}
|
||||
|
||||
$this->logger->info('Media refresh sync completed', $this->stats);
|
||||
|
||||
$this->emit_progress('media_refresh_complete', array(
|
||||
'stats' => $this->stats,
|
||||
));
|
||||
|
||||
} catch (Exception $e) {
|
||||
$this->logger->error('Media refresh sync failed', array('error' => $e->getMessage()));
|
||||
|
||||
if (!$dry_run) {
|
||||
$this->update_sync_state(array(
|
||||
'status' => self::STATUS_FAILED,
|
||||
'last_error' => $e->getMessage(),
|
||||
));
|
||||
}
|
||||
|
||||
return array(
|
||||
'success' => false,
|
||||
'error' => $e->getMessage(),
|
||||
'stats' => $this->stats,
|
||||
);
|
||||
}
|
||||
|
||||
return array(
|
||||
'success' => true,
|
||||
'stats' => $this->stats,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume an interrupted sync
|
||||
*
|
||||
@@ -535,16 +749,24 @@ class MLS_Sync_Engine {
|
||||
private $progress_callback = null;
|
||||
|
||||
/**
|
||||
* Allowed statuses for our database (Active/Pending only)
|
||||
* Allowed statuses for non-HomeProz listings (Active/Pending only)
|
||||
*/
|
||||
const ALLOWED_STATUSES = array('Active', 'Pending');
|
||||
|
||||
/**
|
||||
* Allowed statuses for HomeProz listings (includes Closed for historical records)
|
||||
*/
|
||||
const HOMEPROZ_ALLOWED_STATUSES = array('Active', 'Pending', 'Closed');
|
||||
|
||||
/**
|
||||
* Process a single property record
|
||||
*
|
||||
* During replication, properties are deleted if:
|
||||
* - MlgCanView = false (removed from feed)
|
||||
* - StandardStatus not in (Active, Pending)
|
||||
* - StandardStatus not in allowed list (varies by HomeProz status)
|
||||
*
|
||||
* HomeProz listings are retained even when Closed (sold) for historical viewing.
|
||||
* Non-HomeProz listings are deleted when status is not Active/Pending.
|
||||
*
|
||||
* @param array $property Property data from API
|
||||
* @param bool $dry_run If true, don't make changes
|
||||
@@ -565,8 +787,17 @@ class MLS_Sync_Engine {
|
||||
$can_view = $property['MlgCanView'] ?? true;
|
||||
$status = $property['StandardStatus'] ?? null;
|
||||
|
||||
// Delete if: not viewable OR status is not Active/Pending
|
||||
$should_delete = !$can_view || !in_array($status, self::ALLOWED_STATUSES);
|
||||
// Check if this is a HomeProz listing (by office ID or override list)
|
||||
$listing_id = $property['ListingId'] ?? '';
|
||||
$is_homeproz = (($property['ListOfficeMlsId'] ?? '') === MLS_HOMEPROZ_OFFICE_ID)
|
||||
|| (defined('MLS_HOMEPROZ_OVERRIDE_LISTINGS') && in_array($listing_id, MLS_HOMEPROZ_OVERRIDE_LISTINGS));
|
||||
|
||||
// Determine allowed statuses based on whether it's a HomeProz listing
|
||||
$allowed_statuses = $is_homeproz ? self::HOMEPROZ_ALLOWED_STATUSES : self::ALLOWED_STATUSES;
|
||||
|
||||
// Delete if: not viewable OR status is not in allowed list
|
||||
// HomeProz listings are retained even when Closed (sold)
|
||||
$should_delete = !$can_view || !in_array($status, $allowed_statuses);
|
||||
|
||||
if ($should_delete) {
|
||||
// Check if we have this record locally before attempting delete
|
||||
@@ -609,6 +840,13 @@ class MLS_Sync_Engine {
|
||||
return;
|
||||
}
|
||||
|
||||
// Build spatial location value for the NOT NULL location column
|
||||
$lat = $property['Latitude'] ?? null;
|
||||
$lng = $property['Longitude'] ?? null;
|
||||
$has_coords = ($lat !== null && $lng !== null);
|
||||
$point_lat = $has_coords ? (float) $lat : 0.0;
|
||||
$point_lng = $has_coords ? (float) $lng : 0.0;
|
||||
|
||||
if ($existing) {
|
||||
// Update existing
|
||||
$wpdb->update(
|
||||
@@ -618,33 +856,136 @@ class MLS_Sync_Engine {
|
||||
);
|
||||
$this->stats['updated']++;
|
||||
$this->emit_progress('property_updated', array('listing_key' => $listing_key));
|
||||
} else {
|
||||
// Insert new
|
||||
$data['listing_key'] = $listing_key;
|
||||
$data['created_at'] = current_time('mysql');
|
||||
$wpdb->insert($this->db->properties_table(), $data);
|
||||
$this->stats['created']++;
|
||||
$this->emit_progress('property_created', array('listing_key' => $listing_key));
|
||||
}
|
||||
|
||||
// Update spatial location column (wpdb can't handle ST_PointFromText directly)
|
||||
$lat = $property['Latitude'] ?? null;
|
||||
$lng = $property['Longitude'] ?? null;
|
||||
if ($lat !== null && $lng !== null) {
|
||||
// Update spatial location column (wpdb can't handle ST_PointFromText directly)
|
||||
$wpdb->query($wpdb->prepare(
|
||||
"UPDATE {$this->db->properties_table()} SET location = ST_PointFromText(CONCAT('POINT(', %f, ' ', %f, ')'), 4326) WHERE listing_key = %s",
|
||||
(float) $lat,
|
||||
(float) $lng,
|
||||
$point_lat,
|
||||
$point_lng,
|
||||
$listing_key
|
||||
));
|
||||
} else {
|
||||
// Insert new -- must use raw SQL to include the NOT NULL spatial location column
|
||||
$data['listing_key'] = $listing_key;
|
||||
$data['created_at'] = current_time('mysql');
|
||||
|
||||
$columns = array();
|
||||
$placeholders = array();
|
||||
$values = array();
|
||||
foreach ($data as $col => $val) {
|
||||
$columns[] = "`{$col}`";
|
||||
if ($val === null) {
|
||||
$placeholders[] = 'NULL';
|
||||
} elseif (is_int($val) || is_float($val)) {
|
||||
$placeholders[] = is_int($val) ? '%d' : '%f';
|
||||
$values[] = $val;
|
||||
} else {
|
||||
$placeholders[] = '%s';
|
||||
$values[] = $val;
|
||||
}
|
||||
}
|
||||
|
||||
// Append spatial location column
|
||||
$columns[] = '`location`';
|
||||
$placeholders[] = "ST_PointFromText(CONCAT('POINT(', %f, ' ', %f, ')'), 4326)";
|
||||
$values[] = $point_lat;
|
||||
$values[] = $point_lng;
|
||||
|
||||
$sql = "INSERT INTO {$this->db->properties_table()} (" . implode(', ', $columns) . ") VALUES (" . implode(', ', $placeholders) . ")";
|
||||
$wpdb->query($wpdb->prepare($sql, $values));
|
||||
|
||||
$this->stats['created']++;
|
||||
$this->emit_progress('property_created', array('listing_key' => $listing_key));
|
||||
}
|
||||
|
||||
// Process media if present
|
||||
if (isset($property['Media']) && is_array($property['Media'])) {
|
||||
$this->media_handler->sync_property_media($listing_key, $property['Media'], false, $this->progress_callback);
|
||||
|
||||
// Auto-download and cache all images for HomeProz listings
|
||||
// These images are stored in persistent cache and never garbage collected
|
||||
if ($is_homeproz) {
|
||||
$this->media_handler->download_homeproz_images($listing_key, $this->progress_callback);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download pending media for all HomeProz properties
|
||||
*
|
||||
* Finds HomeProz properties that have media records with pending download status
|
||||
* and downloads them. This ensures HomeProz images are always cached locally.
|
||||
*
|
||||
* @param bool $dry_run If true, don't download
|
||||
* @return array Stats with 'downloaded' and 'skipped' counts
|
||||
*/
|
||||
private function download_pending_homeproz_media($dry_run = false) {
|
||||
global $wpdb;
|
||||
|
||||
$stats = array('downloaded' => 0, 'skipped' => 0, 'properties' => 0);
|
||||
|
||||
// Find HomeProz properties with pending media downloads
|
||||
$properties_table = $this->db->properties_table();
|
||||
$media_table = $this->db->media_table();
|
||||
|
||||
$homeproz_with_pending = $wpdb->get_results(
|
||||
"SELECT DISTINCT p.listing_key
|
||||
FROM {$properties_table} p
|
||||
INNER JOIN {$media_table} m ON p.listing_key = m.listing_key
|
||||
WHERE p.is_homeproz = 1
|
||||
AND m.download_status = 'pending'
|
||||
AND m.media_url IS NOT NULL
|
||||
ORDER BY p.modification_timestamp DESC"
|
||||
);
|
||||
|
||||
if (empty($homeproz_with_pending)) {
|
||||
$this->logger->info('No HomeProz properties with pending media downloads');
|
||||
return $stats;
|
||||
}
|
||||
|
||||
$this->logger->info('Found HomeProz properties with pending media', array(
|
||||
'count' => count($homeproz_with_pending),
|
||||
));
|
||||
|
||||
$this->emit_progress('homeproz_media_start', array(
|
||||
'total_properties' => count($homeproz_with_pending),
|
||||
));
|
||||
|
||||
$property_count = count($homeproz_with_pending);
|
||||
$current = 0;
|
||||
|
||||
foreach ($homeproz_with_pending as $row) {
|
||||
$current++;
|
||||
|
||||
if ($dry_run) {
|
||||
$stats['properties']++;
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->logger->info('Downloading HomeProz media', array(
|
||||
'listing_key' => $row->listing_key,
|
||||
'progress' => "{$current}/{$property_count}",
|
||||
));
|
||||
|
||||
// Download with 10-second delay between each image to respect MLS API limits
|
||||
$result = $this->media_handler->download_homeproz_images(
|
||||
$row->listing_key,
|
||||
$this->progress_callback,
|
||||
10 // delay_seconds between each image
|
||||
);
|
||||
|
||||
$stats['downloaded'] += $result['downloaded'];
|
||||
$stats['skipped'] += $result['skipped'];
|
||||
$stats['properties']++;
|
||||
}
|
||||
|
||||
$this->emit_progress('homeproz_media_complete', $stats);
|
||||
|
||||
$this->logger->info('HomeProz media download completed', $stats);
|
||||
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit progress event
|
||||
*
|
||||
@@ -714,7 +1055,10 @@ class MLS_Sync_Engine {
|
||||
'list_office_key' => $property['ListOfficeKey'] ?? null,
|
||||
'list_office_mls_id' => $property['ListOfficeMlsId'] ?? null,
|
||||
'list_office_name' => $property['ListOfficeName'] ?? null,
|
||||
'is_homeproz' => (($property['ListOfficeMlsId'] ?? '') === MLS_HOMEPROZ_OFFICE_ID) ? 1 : 0,
|
||||
'is_homeproz' => (
|
||||
(($property['ListOfficeMlsId'] ?? '') === MLS_HOMEPROZ_OFFICE_ID)
|
||||
|| (defined('MLS_HOMEPROZ_OVERRIDE_LISTINGS') && in_array($property['ListingId'] ?? '', MLS_HOMEPROZ_OVERRIDE_LISTINGS))
|
||||
) ? 1 : 0,
|
||||
|
||||
'photos_count' => $property['PhotosCount'] ?? 0,
|
||||
'modification_timestamp' => $this->format_timestamp($property['ModificationTimestamp'] ?? null),
|
||||
@@ -832,7 +1176,11 @@ class MLS_Sync_Engine {
|
||||
);
|
||||
|
||||
if ($timestamp) {
|
||||
// Look back 10 minutes past the latest timestamp as a safety margin
|
||||
// to catch any records that may have been missed due to race conditions
|
||||
// or clock skew between our DB and the MLS API
|
||||
$dt = new DateTime($timestamp);
|
||||
$dt->modify('-10 minutes');
|
||||
return $dt->format('Y-m-d\TH:i:s.v\Z');
|
||||
}
|
||||
|
||||
@@ -992,7 +1340,13 @@ class MLS_Sync_Engine {
|
||||
// Step 2: Check if a sync is actively running
|
||||
$running = $this->get_running_sync();
|
||||
if ($running) {
|
||||
$status("Sync #{$running->id} is already running (started {$running->started_at})", 'warning');
|
||||
// If a full sync is in progress, exit silently so cron incremental
|
||||
// syncs don't log warnings while the weekly full sync runs
|
||||
if ($running->sync_type === 'full') {
|
||||
$status("Full sync #{$running->id} in progress (started {$running->started_at}), skipping", 'info');
|
||||
} else {
|
||||
$status("Sync #{$running->id} is already running (started {$running->started_at})", 'warning');
|
||||
}
|
||||
return array(
|
||||
'success' => false,
|
||||
'action' => 'aborted',
|
||||
|
||||
Reference in New Issue
Block a user