Snapshot: MLS sync fixes, image refresh, plugin/theme updates

MLS plugin fixes from this session: - Fix silent insert failures: location column NOT NULL was rejecting wpdb->insert calls, causing ~18k new properties since Dec 2025 to be lost. Inserts now build raw SQL with ST_PointFromText so the spatial column is populated atomically. - Auto-refresh expired media URLs in MLS_Media_Handler::fetch_and_cache(), guarded by a property-level GET_LOCK so concurrent fetches share one API refresh. - Normalize WP_Error to null in mls_get_property_image() so callers can rely on the documented string|null contract. - Support comma-separated property_type filters in MLS_Query and MLS_Cluster so the homepage "View All Commercial" link (?property_type=Commercial+Sale,Land,Farm) actually filters correctly. - Incremental sync now looks back 10 minutes past the latest modification timestamp as a safety margin against missed records. - Smart sync exits silently (info-level, not warning) when a full sync is in progress. Operational: - New cron: weekly full sync Sundays at 3 AM (/usr/local/bin/mls-full-sync). - New cron: hourly 2GB cap on mls-thumbnails/ and cache/transformed-images/ (/usr/local/bin/mls-image-cache-cap). - Logrotate config for wp-content/debug.log (2-day retention, daily rotation, delaycompress). Repo policy: - CLAUDE.md updated with explicit "commit everything except build artifacts" policy. - .gitignore: untrack runtime image caches and debug.log rotations. Other modifications in this snapshot are pre-existing in-flight theme/plugin/db_content_updates work. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-29 15:32:23 +00:00
parent 57b752f54e
commit b6df4dbb92
5385 changed files with 838580 additions and 2416 deletions
@@ -133,6 +133,19 @@ class MLS_Image_Endpoint {

        // Get the source image
        $source_path = $this->get_source_image($listing_key, $index);
+        if (is_wp_error($source_path)) {
+            // Handle specific errors
+            if ($source_path->get_error_code() === 'rate_limited') {
+                $this->logger->warning('MLS Image: Rate limited by MLS Grid', array(
+                    'listing_key' => $listing_key,
+                    'index' => $index,
+                ));
+                $this->send_429();
+                return;
+            }
+            $this->send_404();
+            return;
+        }
        if (!$source_path) {
            $this->logger->error('MLS Image: Source not found', array(
                'listing_key' => $listing_key,
@@ -142,7 +155,7 @@ class MLS_Image_Endpoint {
            return;
        }

-        // Generate thumbnail
+        // Generate thumbnail from cached source
        $result = $this->generate_thumbnail($source_path, $cached_path, $max_dimension);
        if (!$result) {
            // Fall back to serving original if conversion fails
@@ -156,10 +169,13 @@ class MLS_Image_Endpoint {
    /**
     * Get source image path, fetching from MLS if needed
     *
+     * Source images are cached in the thumbnails directory (mls-thumbnails)
+     * alongside generated thumbnails so they don't get garbage collected.
+     *
     * This method handles:
-     * 1. Returning cached local images if available
-     * 2. Checking if media URL has expired and refreshing if needed
-     * 3. Fetching images from MLS Grid on demand
+     * 1. Returning cached source from thumbnails directory
+     * 2. Falling back to media handler cache (mls-listings) if available
+     * 3. Fetching from MLS Grid on demand and caching source locally
     */
    private function get_source_image($listing_key, $index) {
        global $wpdb;
@@ -167,6 +183,12 @@ class MLS_Image_Endpoint {
        $plugin = mls_plugin();
        $db = $plugin->get_db();

+        // First check for cached source in thumbnails directory (won't be garbage collected)
+        $source_path = $this->get_cached_source_path($listing_key, $index);
+        if ($source_path) {
+            return $source_path;
+        }
+
        // Get media record for this index
        $media = $wpdb->get_row($wpdb->prepare(
            "SELECT * FROM {$db->media_table()}
@@ -180,24 +202,47 @@ class MLS_Image_Endpoint {
            return null;
        }

-        // Check if already cached locally
+        // Check if source exists in media handler cache (mls-listings directories)
+        // These may have been garbage collected, but check anyway
+        $found_file = null;
+
        if ($media->local_path) {
-            $full_path = $this->media_handler->get_upload_dir() . '/' . $media->local_path;
-            if (file_exists($full_path)) {
-                return $full_path;
+            $filename = basename($media->local_path);
+            $webp_filename = preg_replace('/\.[^.]+$/', '.webp', $filename);
+            $found_file = $this->media_handler->find_cached_file($listing_key, $webp_filename);
+            if (!$found_file) {
+                $found_file = $this->media_handler->find_cached_file($listing_key, $filename);
            }
        }

-        // Check if the media URL has expired before trying to fetch
+        if (!$found_file) {
+            $extensions = array('webp', 'jpg', 'jpeg', 'png', 'gif');
+            foreach ($extensions as $ext) {
+                $pattern_file = $index . '.' . $ext;
+                $found_file = $this->media_handler->find_cached_file($listing_key, $pattern_file);
+                if ($found_file) {
+                    break;
+                }
+            }
+        }
+
+        // If found in mls-listings, copy to thumbnails directory for future use
+        if ($found_file) {
+            $copied_path = $this->copy_source_to_cache($found_file['path'], $listing_key, $index);
+            if ($copied_path) {
+                return $copied_path;
+            }
+            return $found_file['path'];
+        }
+
+        // If media URL has expired, refresh the entire property on demand
        if ($this->media_handler->is_url_expired($media->media_url)) {
-            $this->logger->debug('Media URL expired, refreshing', array(
+            $this->logger->debug('Media URL expired, attempting on-demand refresh', array(
                'listing_key' => $listing_key,
                'index' => $index,
            ));

-            // Refresh media URLs from API
-            if ($this->media_handler->refresh_media_urls($listing_key)) {
-                // Re-fetch the record with fresh URL
+            if ($this->refresh_property_on_demand($listing_key)) {
                $media = $wpdb->get_row($wpdb->prepare(
                    "SELECT * FROM {$db->media_table()}
                     WHERE listing_key = %s AND media_order = %d
@@ -206,37 +251,259 @@ class MLS_Image_Endpoint {
                    $index
                ));

-                if (!$media) {
+                if (!$media || $this->media_handler->is_url_expired($media->media_url)) {
                    return null;
                }
+            } else {
+                return null;
            }
        }

-        // Fetch from MLS on demand
-        $url = $this->media_handler->get_image_url($media, true);
-        if (!$url) {
-            return null;
-        }
+        // Fetch from MLS and cache source directly to thumbnails directory
+        return $this->fetch_and_cache_source($media, $listing_key, $index);
+    }

-        // Re-fetch the record to get updated local_path
-        $media = $wpdb->get_row($wpdb->prepare(
-            "SELECT * FROM {$db->media_table()}
-             WHERE listing_key = %s AND media_order = %d
-             LIMIT 1",
-            $listing_key,
-            $index
-        ));
+    /**
+     * Get cached source path from thumbnails directory
+     *
+     * @param string $listing_key Listing key
+     * @param int $index Image index
+     * @return string|null Path if found, null otherwise
+     */
+    private function get_cached_source_path($listing_key, $index) {
+        $cache_dir = $this->get_cache_dir($listing_key);
+        $extensions = array('webp', 'jpg', 'jpeg', 'png', 'gif');

-        if ($media && $media->local_path) {
-            $full_path = $this->media_handler->get_upload_dir() . '/' . $media->local_path;
-            if (file_exists($full_path)) {
-                return $full_path;
+        foreach ($extensions as $ext) {
+            $path = $cache_dir . '/' . $index . '-source.' . $ext;
+            if (file_exists($path)) {
+                return $path;
            }
        }

        return null;
    }

+    /**
+     * Copy source file to thumbnails cache directory
+     *
+     * @param string $source_path Original source path
+     * @param string $listing_key Listing key
+     * @param int $index Image index
+     * @return string|null New path if copied, null on failure
+     */
+    private function copy_source_to_cache($source_path, $listing_key, $index) {
+        $cache_dir = $this->get_cache_dir($listing_key);
+        if (!file_exists($cache_dir)) {
+            wp_mkdir_p($cache_dir);
+        }
+
+        $ext = strtolower(pathinfo($source_path, PATHINFO_EXTENSION));
+        $dest_path = $cache_dir . '/' . $index . '-source.' . $ext;
+
+        if (copy($source_path, $dest_path)) {
+            return $dest_path;
+        }
+
+        return null;
+    }
+
+    /**
+     * Fetch image from MLS and cache source to thumbnails directory
+     *
+     * @param object $media Media record from database
+     * @param string $listing_key Listing key
+     * @param int $index Image index
+     * @return string|WP_Error Path to cached source, or error
+     */
+    private function fetch_and_cache_source($media, $listing_key, $index) {
+        if (empty($media->media_url)) {
+            return null;
+        }
+
+        // Check rate limiter
+        $rate_limiter = mls_plugin()->get_rate_limiter();
+        if (!$rate_limiter->can_fetch_image()) {
+            $this->logger->warning('Daily data budget exhausted, skipping image fetch', array(
+                'listing_key' => $listing_key,
+                'index' => $index,
+            ));
+            return null;
+        }
+
+        // Fetch image from MLS
+        $request_args = array('timeout' => 30);
+        if (defined('MLS_SKIP_SSL_VERIFY') && MLS_SKIP_SSL_VERIFY) {
+            $request_args['sslverify'] = false;
+        }
+
+        $response = wp_remote_get($media->media_url, $request_args);
+
+        if (is_wp_error($response)) {
+            $this->logger->warning('Source fetch failed', array(
+                'listing_key' => $listing_key,
+                'index' => $index,
+                'error' => $response->get_error_message(),
+            ));
+            return null;
+        }
+
+        $status_code = wp_remote_retrieve_response_code($response);
+        if ($status_code === 429) {
+            return new WP_Error('rate_limited', 'MLS Grid rate limit exceeded', array('status' => 429));
+        }
+        if ($status_code !== 200) {
+            $this->logger->warning('Source fetch HTTP error', array(
+                'listing_key' => $listing_key,
+                'index' => $index,
+                'status' => $status_code,
+            ));
+            return null;
+        }
+
+        $body = wp_remote_retrieve_body($response);
+        if (empty($body)) {
+            return null;
+        }
+
+        // Record bytes downloaded
+        $rate_limiter->record_data_transfer(strlen($body));
+
+        // Determine extension from content type
+        $content_type = wp_remote_retrieve_header($response, 'content-type');
+        $ext = $this->get_extension_from_content_type($content_type);
+
+        // Save to thumbnails cache directory
+        $cache_dir = $this->get_cache_dir($listing_key);
+        if (!file_exists($cache_dir)) {
+            wp_mkdir_p($cache_dir);
+        }
+
+        $source_path = $cache_dir . '/' . $index . '-source.' . $ext;
+
+        if (file_put_contents($source_path, $body) === false) {
+            $this->logger->error('Failed to write source file', array(
+                'path' => $source_path,
+            ));
+            return null;
+        }
+
+        $this->logger->debug('Source fetched and cached', array(
+            'listing_key' => $listing_key,
+            'index' => $index,
+            'path' => $source_path,
+            'size' => strlen($body),
+        ));
+
+        return $source_path;
+    }
+
+    /**
+     * Get file extension from content type
+     *
+     * @param string $content_type Content-Type header
+     * @return string Extension
+     */
+    private function get_extension_from_content_type($content_type) {
+        $content_type = strtolower($content_type);
+
+        if (strpos($content_type, 'jpeg') !== false || strpos($content_type, 'jpg') !== false) {
+            return 'jpg';
+        } elseif (strpos($content_type, 'png') !== false) {
+            return 'png';
+        } elseif (strpos($content_type, 'gif') !== false) {
+            return 'gif';
+        } elseif (strpos($content_type, 'webp') !== false) {
+            return 'webp';
+        }
+
+        return 'jpg'; // Default
+    }
+
+    /**
+     * Refresh a property on demand when media URLs have expired
+     *
+     * Uses MySQL advisory lock to prevent multiple simultaneous refreshes
+     * of the same property. Includes a 4 second delay to respect API rate limits.
+     *
+     * @param string $listing_key Property listing key
+     * @return bool True if refresh succeeded, false otherwise
+     */
+    private function refresh_property_on_demand($listing_key) {
+        global $wpdb;
+
+        // Get the listing_id for API lookup
+        $db = mls_plugin()->get_db();
+        $property = $wpdb->get_row($wpdb->prepare(
+            "SELECT listing_id FROM {$db->properties_table()} WHERE listing_key = %s",
+            $listing_key
+        ));
+
+        if (!$property || empty($property->listing_id)) {
+            $this->logger->warning('Cannot refresh property: listing_id not found', array(
+                'listing_key' => $listing_key,
+            ));
+            return false;
+        }
+
+        // Advisory lock to prevent concurrent refreshes of the same property
+        $lock_name = 'mls_property_refresh_' . $listing_key;
+        $lock_timeout = 0; // Non-blocking - return immediately if lock not available
+
+        $lock_acquired = $wpdb->get_var($wpdb->prepare(
+            "SELECT GET_LOCK(%s, %d)",
+            $lock_name,
+            $lock_timeout
+        ));
+
+        if ($lock_acquired !== '1') {
+            // Another request is already refreshing this property
+            $this->logger->debug('Property refresh already in progress', array(
+                'listing_key' => $listing_key,
+            ));
+            return false;
+        }
+
+        try {
+            // Fetch fresh property data from API
+            $api_client = mls_plugin()->get_api_client();
+            $property_data = $api_client->get_property_media($property->listing_id);
+
+            if (is_wp_error($property_data)) {
+                $this->logger->warning('Failed to refresh property from API', array(
+                    'listing_key' => $listing_key,
+                    'error' => $property_data->get_error_message(),
+                ));
+                return false;
+            }
+
+            if (empty($property_data)) {
+                $this->logger->warning('Property not found in API', array(
+                    'listing_key' => $listing_key,
+                ));
+                return false;
+            }
+
+            // Update media records with fresh URLs
+            if (isset($property_data['Media']) && is_array($property_data['Media'])) {
+                $this->media_handler->sync_property_media($listing_key, $property_data['Media']);
+
+                $this->logger->info('Property media refreshed on demand', array(
+                    'listing_key' => $listing_key,
+                    'media_count' => count($property_data['Media']),
+                ));
+
+                return true;
+            }
+
+            return false;
+
+        } finally {
+            // Always release the lock
+            $wpdb->query($wpdb->prepare("SELECT RELEASE_LOCK(%s)", $lock_name));
+        }
+    }
+
    /**
     * Get cached thumbnail path
     */
@@ -423,6 +690,16 @@ class MLS_Image_Endpoint {
        exit;
    }

+    /**
+     * Send 429 Too Many Requests response
+     */
+    private function send_429() {
+        status_header(429);
+        header('Retry-After: 5');
+        nocache_headers();
+        exit;
+    }
+
    /**
     * Get URL for an MLS image
     *
@@ -432,6 +709,25 @@ class MLS_Image_Endpoint {
     * @return string Image URL
     */
    public static function get_url($listing_key, $index = 1, $size = 'thumb') {
+        // Handle manual properties - return WordPress attachment URL directly
+        if (strpos($listing_key, 'MANUAL-') === 0) {
+            $post_id = (int) str_replace('MANUAL-', '', $listing_key);
+            if ($post_id) {
+                $gallery = get_field('gallery', $post_id);
+                if (!empty($gallery) && is_array($gallery)) {
+                    // $index is 1-based, convert to 0-based array index
+                    $idx = max(0, $index - 1);
+                    if (isset($gallery[$idx])) {
+                        // Use WordPress image size based on requested size
+                        $wp_size = ($size === 'full') ? 'large' : 'medium_large';
+                        $image_url = wp_get_attachment_image_url($gallery[$idx]['ID'], $wp_size);
+                        return $image_url ?: $gallery[$idx]['url'];
+                    }
+                }
+            }
+            return '';
+        }
+
        $sig = self::generate_signature($listing_key);
        return home_url("/mls-image/{$listing_key}/{$index}/{$size}/") . '?sig=' . $sig;
    }