From b9cddd2f64e921819c40eae23fdf8f05d7a004d5 Mon Sep 17 00:00:00 2001 From: "Hanson.xyz Dev" Date: Mon, 15 Dec 2025 08:25:37 -0600 Subject: [PATCH] Refactor MLS sync to Active/Pending only with on-demand media Major changes to sync strategy following MLS Grid best practices: - Initial sync now fetches only Active/Pending properties (~30K vs 1.3M) - Replication (incremental) fetches all changes, deletes non-Active/Pending - On-demand media fetching replaces background queue (avoids rate limits) - Media downloaded and cached when first viewed, not during sync - Updated CLI commands: wp mls media status/fetch/clear - Comprehensive documentation with troubleshooting guide This fixes the "Value out of range" API error caused by high $skip values. Co-Authored-By: Claude --- .../mls-by-hansonxyz/cli/class-mls-cli.php | 284 ++----- .../plugins/mls-by-hansonxyz/docs/CLAUDE.md | 175 ++-- .../includes/class-mls-api-client.php | 45 +- .../includes/class-mls-media-handler.php | 799 +++++------------- .../includes/class-mls-sync-engine.php | 66 +- .../mls-by-hansonxyz/mls-by-hansonxyz.php | 43 +- 6 files changed, 538 insertions(+), 874 deletions(-) diff --git a/wp-content/plugins/mls-by-hansonxyz/cli/class-mls-cli.php b/wp-content/plugins/mls-by-hansonxyz/cli/class-mls-cli.php index 476c0ba9..16ec9fba 100644 --- a/wp-content/plugins/mls-by-hansonxyz/cli/class-mls-cli.php +++ b/wp-content/plugins/mls-by-hansonxyz/cli/class-mls-cli.php @@ -338,14 +338,12 @@ class MLS_CLI { break; case 'media': - // Redirect to the new media command - WP_CLI::line('Note: "wp mls sync media" is deprecated. Use "wp mls media process" instead.'); + // Media is now on-demand, this sync type is deprecated + WP_CLI::line('Note: "wp mls sync media" is deprecated.'); + WP_CLI::line('Media is now fetched on-demand when properties are viewed on the website.'); WP_CLI::line(''); - $this->media(array('process'), array( - 'limit' => $limit ?: 100, - 'verbose' => $verbose, - 'quiet' => $quiet, - )); + WP_CLI::line('Use "wp mls media status" to see cache statistics.'); + WP_CLI::line('Use "wp mls media fetch --listing=" to pre-cache a specific listing.'); break; case 'resume': @@ -799,263 +797,119 @@ class MLS_CLI { } /** - * Manage media download queue. + * Show media cache status and manage cached files. + * + * Media is now fetched on-demand when properties are viewed on the website. + * This command shows cache statistics and allows management of cached files. * * ## OPTIONS * - * - * : Action: queue, process, status, reset, logs + * [] + * : Action: status (default), fetch, clear + * + * [--listing=] + * : Listing key for fetch or clear actions * * [--limit=] - * : Limit number of items to process - * - * [--verbose] - * : Show detailed output - * - * [--quiet] - * : Suppress progress output - * - * [--days=] - * : Days of logs to keep (for logs --clear) - * - * [--clear] - * : Clear logs older than --days + * : For fetch action, max images to fetch (default: 1) * * ## EXAMPLES * - * wp mls media status # Show queue statistics - * wp mls media process # Process pending downloads (rate limited) - * wp mls media process --limit=50 # Process up to 50 items - * wp mls media reset # Reset failed downloads for retry - * wp mls media logs # Show recent download logs - * wp mls media logs --clear --days=7 # Clear logs older than 7 days + * wp mls media status # Show cache statistics + * wp mls media fetch --listing=NST123456 # Fetch images for a listing + * wp mls media fetch --listing=NST123456 --limit=10 # Fetch up to 10 images + * wp mls media clear --listing=NST123456 # Clear cached images for a listing * * @subcommand media */ public function media($args, $assoc_args) { $action = isset($args[0]) ? $args[0] : 'status'; - $limit = isset($assoc_args['limit']) ? (int) $assoc_args['limit'] : 100; - $verbose = isset($assoc_args['verbose']); - $quiet = isset($assoc_args['quiet']); $media_handler = $this->plugin->get_media_handler(); switch ($action) { case 'status': - case 'queue': - $stats = $media_handler->get_queue_stats(); + $stats = $media_handler->get_cache_stats(); WP_CLI::line(''); - WP_CLI::line('=== Media Download Queue ==='); + WP_CLI::line('=== Media Cache Status ==='); WP_CLI::line(''); - WP_CLI::line(sprintf('Pending total: %d', $stats['pending'])); - WP_CLI::line(sprintf('Ready now: %d', $stats['ready'])); - WP_CLI::line(sprintf('In backoff: %d (retry scheduled)', $stats['in_backoff'])); - WP_CLI::line(sprintf('Failed: %d (max attempts reached)', $stats['failed'])); - WP_CLI::line(sprintf('Completed: %d', $stats['completed'])); + WP_CLI::line(sprintf('Total media records: %d', $stats['total_media'])); + WP_CLI::line(sprintf('Cached locally: %d', $stats['cached'])); + WP_CLI::line(sprintf('Not yet cached: %d', $stats['uncached'])); WP_CLI::line(''); - if ($stats['ready'] > 0) { - WP_CLI::line(sprintf( - 'Run "wp mls media process --limit=%d" to download pending media.', - min($stats['ready'], 100) - )); - WP_CLI::line(sprintf( - 'Estimated time: %d minutes (at 700ms per image)', - ceil($stats['ready'] * 0.7 / 60) - )); - } - - if ($stats['failed'] > 0) { - WP_CLI::line(''); - WP_CLI::line('Run "wp mls media reset" to retry failed downloads.'); - } + $cache_percent = $stats['total_media'] > 0 + ? round(($stats['cached'] / $stats['total_media']) * 100, 1) + : 0; + WP_CLI::line(sprintf('Cache rate: %.1f%%', $cache_percent)); + WP_CLI::line(''); + WP_CLI::line('Images are fetched on-demand when properties are viewed.'); + WP_CLI::line('Use "wp mls media fetch --listing=" to pre-cache specific listings.'); WP_CLI::line(''); break; - case 'process': - $stats = $media_handler->get_queue_stats(); - - if ($stats['ready'] === 0) { - WP_CLI::success('No media ready to download.'); - break; + case 'fetch': + $listing_key = isset($assoc_args['listing']) ? $assoc_args['listing'] : null; + if (!$listing_key) { + WP_CLI::error('Please specify --listing='); } - $process_count = min($limit, $stats['ready']); + $limit = isset($assoc_args['limit']) ? (int) $assoc_args['limit'] : 1; - WP_CLI::line(sprintf( - 'Processing %d media items (rate limited: 1 per 700ms)...', - $process_count - )); - WP_CLI::line(sprintf( - 'Estimated time: %d minutes', - ceil($process_count * 0.7 / 60) - )); + WP_CLI::line(sprintf('Fetching up to %d images for listing %s...', $limit, $listing_key)); - if (!$quiet) { - WP_CLI::line('Legend: P=downloaded B=backoff (retry later) E=error'); - echo "\n"; - } + $images = $media_handler->get_listing_images($listing_key, $limit); - // Progress callback - $progress_callback = null; - if (!$quiet) { - $progress_callback = function($event, $data = array()) use ($verbose) { - if ($verbose) { - $this->output_verbose_media_event($event, $data); - } else { - switch ($event) { - case 'media_downloaded': - echo 'P'; - break; - case 'media_backoff': - echo 'B'; - break; - case 'media_error': - echo 'E'; - break; - } - } - }; - } - - $result = $media_handler->process_queue($process_count, $progress_callback); - - if (!$quiet) { - echo "\n\n"; + $cached_count = 0; + foreach ($images as $img) { + if ($img->local_url) { + $cached_count++; + } } WP_CLI::line(sprintf( - 'Results: %d success, %d backoff, %d failed out of %d processed', - $result['success'], - $result['skipped'], - $result['failed'], - $result['processed'] + 'Result: %d/%d images now cached for this listing.', + $cached_count, + count($images) )); - // Show updated stats - $new_stats = $media_handler->get_queue_stats(); - WP_CLI::line(sprintf('Queue remaining: %d ready, %d in backoff', $new_stats['ready'], $new_stats['in_backoff'])); - - if ($result['failed'] > 0 || $result['skipped'] > 0) { - WP_CLI::line(''); - WP_CLI::line('Items in backoff will be retried after 3 hours.'); - WP_CLI::line('Run "wp mls media logs" to see download history.'); - } - - if ($result['success'] > 0) { - WP_CLI::success('Media processing complete.'); - } - break; - - case 'reset': - WP_CLI::line('Resetting failed downloads for retry...'); - - $reset_count = $media_handler->reset_failed_downloads(); - - if ($reset_count > 0) { - WP_CLI::success(sprintf('Reset %d failed downloads. They will be retried on next process.', $reset_count)); + if ($cached_count > 0) { + WP_CLI::success('Images fetched successfully.'); + } elseif (count($images) === 0) { + WP_CLI::warning('No media records found for this listing.'); } else { - WP_CLI::success('No failed downloads to reset.'); + WP_CLI::warning('Failed to fetch images. Check logs for details.'); } break; - case 'logs': - if (isset($assoc_args['clear'])) { - $days = isset($assoc_args['days']) ? (int) $assoc_args['days'] : 7; - $deleted = $media_handler->clear_old_logs($days); - WP_CLI::success(sprintf('Deleted %d log entries older than %d days.', $deleted, $days)); - break; + case 'clear': + $listing_key = isset($assoc_args['listing']) ? $assoc_args['listing'] : null; + if (!$listing_key) { + WP_CLI::error('Please specify --listing=. To clear all media, use "wp mls cache clear --confirm".'); } - $logs = $media_handler->get_download_logs($limit); + // Just clear the local files, keep metadata + global $wpdb; + $listing_dir = $media_handler->get_listing_dir($listing_key); - if (empty($logs)) { - WP_CLI::success('No download logs found.'); - break; + if (is_dir($listing_dir)) { + $this->recursive_delete($listing_dir); } - WP_CLI::line(''); - WP_CLI::line('=== Recent Download Logs ==='); - WP_CLI::line(''); + // Clear local_path and local_url but keep the records + $wpdb->query($wpdb->prepare( + "UPDATE {$this->plugin->get_db()->media_table()} + SET local_path = NULL, local_url = NULL, downloaded_at = NULL + WHERE listing_key = %s", + $listing_key + )); - foreach ($logs as $log) { - $status_indicator = ''; - switch ($log->action) { - case 'success': - $status_indicator = '[OK]'; - break; - case 'rate_limited': - $status_indicator = '[429]'; - break; - case 'permanent_error': - $status_indicator = '[ERR]'; - break; - case 'error': - $status_indicator = '[FAIL]'; - break; - default: - $status_indicator = "[{$log->action}]"; - } - - $line = sprintf( - '%s %s %s %s %dms', - $log->created_at, - $status_indicator, - $log->listing_key, - $log->media_key, - $log->response_time_ms - ); - - if ($log->status_code) { - $line .= " HTTP:{$log->status_code}"; - } - - if ($log->error_message) { - $line .= " - {$log->error_message}"; - } - - WP_CLI::line($line); - } - - WP_CLI::line(''); - WP_CLI::line(sprintf('Showing %d most recent entries. Use --limit=N to see more.', count($logs))); - WP_CLI::line(''); + WP_CLI::success(sprintf('Cleared cached images for listing %s. They will be re-fetched on demand.', $listing_key)); break; default: - WP_CLI::error("Unknown action: {$action}. Use 'status', 'process', 'reset', or 'logs'."); - } - } - - /** - * Output verbose media event information - * - * @param string $event Event name - * @param array $data Event data - */ - private function output_verbose_media_event($event, $data) { - $timestamp = date('H:i:s'); - - switch ($event) { - case 'media_downloaded': - $listing = $data['listing_key'] ?? 'unknown'; - $key = $data['media_key'] ?? 'unknown'; - WP_CLI::line("[{$timestamp}] DOWNLOADED: {$listing} / {$key}"); - break; - - case 'media_backoff': - $listing = $data['listing_key'] ?? 'unknown'; - $key = $data['media_key'] ?? 'unknown'; - WP_CLI::warning("[{$timestamp}] BACKOFF: {$listing} / {$key} - will retry in 3 hours"); - break; - - case 'media_error': - $listing = $data['listing_key'] ?? 'unknown'; - $key = $data['media_key'] ?? 'unknown'; - $error = $data['error'] ?? 'Unknown error'; - WP_CLI::error("[{$timestamp}] ERROR: {$listing} / {$key} - {$error}", false); - break; + WP_CLI::error("Unknown action: {$action}. Use 'status', 'fetch', or 'clear'."); } } diff --git a/wp-content/plugins/mls-by-hansonxyz/docs/CLAUDE.md b/wp-content/plugins/mls-by-hansonxyz/docs/CLAUDE.md index 7b919d7b..d794b641 100644 --- a/wp-content/plugins/mls-by-hansonxyz/docs/CLAUDE.md +++ b/wp-content/plugins/mls-by-hansonxyz/docs/CLAUDE.md @@ -17,9 +17,8 @@ All tables use `{$wpdb->prefix}mls_` prefix: | Table | Purpose | |-------|---------| -| `mls_properties` | Listing data | -| `mls_media` | Media files with download queue | -| `mls_media_log` | Media download attempt history | +| `mls_properties` | Listing data (Active/Pending only) | +| `mls_media` | Media metadata and cache status | | `mls_sync_state` | Sync progress tracking | | `mls_rate_limits` | API usage tracking | | `mls_sync_log` | Debug logging | @@ -40,7 +39,7 @@ MUST comply with these limits: - 40,000 requests/day - 4GB data/hour -Media downloads use 700ms delay (25% buffer) between requests. +**Important**: The API rejects `$skip` values over ~80,000. Always use `@odata.nextLink` for pagination, never manual `$skip`. ### Key Files @@ -48,7 +47,7 @@ Media downloads use 700ms delay (25% buffer) between requests. |------|---------| | `includes/class-mls-api-client.php` | API communication, auth, gzip | | `includes/class-mls-sync-engine.php` | Sync orchestration | -| `includes/class-mls-media-handler.php` | Media queue and download | +| `includes/class-mls-media-handler.php` | On-demand media fetch and cache | | `includes/class-mls-query.php` | Public query API | | `includes/class-mls-rate-limiter.php` | Rate limit compliance | | `cli/class-mls-cli.php` | WP-CLI commands | @@ -64,18 +63,16 @@ wp mls test auth wp mls status wp mls status rate-limits -# Run property sync (queues media, does not download) -wp mls sync full [--dry-run] [--limit=N] [--verbose] -wp mls sync incremental [--dry-run] [--verbose] +# Run property sync +wp mls sync full [--dry-run] [--limit=N] [--verbose] # Initial: Active/Pending only +wp mls sync incremental [--dry-run] [--verbose] # Replication: all changes wp mls sync resume --id= -# Media download queue (separate from property sync) -wp mls media status # Show queue stats -wp mls media process # Download queued media (rate limited) -wp mls media process --limit=50 --verbose -wp mls media reset # Reset failed downloads for retry -wp mls media logs # View download history -wp mls media logs --clear --days=7 +# Media cache (images fetched on-demand when viewed) +wp mls media status # Show cache statistics +wp mls media fetch --listing= # Pre-cache images for a listing +wp mls media fetch --listing= --limit=10 # Fetch up to 10 images +wp mls media clear --listing= # Clear cached images for re-fetch # Statistics wp mls stats @@ -83,9 +80,6 @@ wp mls stats # Cache management wp mls cache clear --confirm wp mls cache cleanup -wp mls cache missing # View failed media downloads -wp mls cache missing --limit=20 # View first 20 entries -wp mls cache missing --clear # Clear the log # Recovery commands wp mls recovery list # Show resumable syncs @@ -93,26 +87,64 @@ wp mls recovery auto # Auto-resume most recent failed sync wp mls recovery cleanup # Mark stale (>1hr) syncs as failed ``` -### Media Queue System +## Sync Strategy (IMPORTANT) -Media downloads are now queue-based and separate from property sync: +The sync follows MLS Grid best practices for replication: -1. **Property sync** (`wp mls sync full/incremental`) queues media records -2. **Media process** (`wp mls media process`) downloads queued media with rate limiting -3. Downloads are rate-limited to 700ms between requests (under 2/sec limit) -4. Failed downloads get 3-hour backoff before retry -5. After 5 attempts, items are marked failed and logged +### Initial Import (`wp mls sync full`) -**Queue states:** -- `pending` - Ready for download -- `completed` - Successfully downloaded -- `failed` - Max attempts reached +- Fetches ONLY `Active` and `Pending` properties +- Filter: `MlgCanView eq true and (StandardStatus eq 'Active' or StandardStatus eq 'Pending')` +- Uses `@odata.nextLink` for pagination (NOT `$skip`) +- Stores media metadata but does NOT download images +- ~30,000 records for NorthStar MLS (vs 1.3M total including Closed) -**Media table columns:** -- `download_status` - pending/completed/failed -- `retry_after` - Next retry time (3hr backoff on rate limit) -- `queued_at` - When item was queued -- `download_attempts` - Attempt count (max 5) +### Replication (`wp mls sync incremental`) + +- Fetches ALL properties modified since last sync +- NO filter on `MlgCanView` or `StandardStatus` - we need to see changes +- For each record received: + - If `MlgCanView = false` -> DELETE from local DB + - If `StandardStatus` not in (Active, Pending) -> DELETE from local DB + - Otherwise -> INSERT or UPDATE +- This handles: new listings, price changes, status changes (Active->Sold), removals + +### Why This Approach? + +1. **MLS Grid API limits `$skip` to ~80,000** - bulk scanning all 1.3M records fails +2. **We only care about available properties** - no need to store Closed/Sold +3. **Replication is efficient** - only fetches changed records +4. **Proper deletion handling** - when a property sells, we remove it + +### Data Flow + +``` +Initial Import: + API (Active/Pending + MlgCanView=true) -> Local DB + +Replication (every 15 min): + API (ModificationTimestamp > last_sync) -> Check each record: + - MlgCanView=false OR Status!=Active/Pending -> DELETE locally + - Otherwise -> UPSERT locally +``` + +## Media System (On-Demand Fetching) + +Per MLS Grid rules, media URLs must NOT be used directly on websites. Images must be downloaded and served from our own server. + +**How it works:** +1. **Property sync** stores media metadata (URLs, keys, order) but does NOT download images +2. **On-demand fetch**: When `mls_get_property_image()` is called, the image is fetched and cached locally +3. **Subsequent requests** serve from local cache +4. **Pre-caching**: Use `wp mls media fetch --listing=` to pre-cache specific listings + +**Benefits:** +- No rate limit issues from bulk downloading +- Images cached only when needed (saves bandwidth/storage) +- Automatic re-fetch if cache is cleared +- Works with MLS Grid's image URL expiration + +**Cache location:** `wp-content/uploads/mls-listings/{prefix}/{listing_key}/` ### Progress Output @@ -121,23 +153,10 @@ Property sync (compact mode): - `#` = property updated - `x` = property deleted - `-` = skipped (dry-run) -- `q` = media queued -- `p` = media skipped (already downloaded) - `|` = page complete -Media process (compact mode): -- `P` = downloaded -- `B` = backoff (retry later) -- `E` = error - With --verbose: Full timestamped output. -### Missing Media Log - -Permanently failed media downloads logged to: `wp-content/uploads/mls-missing-media.log` - -Format: `[timestamp] listing_key | media_key | error | url` - ### Sync Recovery The sync engine saves progress after each page: @@ -152,16 +171,15 @@ The sync engine saves progress after each page: ### Recommended Cron Setup ```bash -# Property sync every 30 minutes -*/30 * * * * cd /var/www/html && wp mls recovery auto --quiet && wp mls sync incremental --allow-root >> /var/log/mls-sync.log 2>&1 +# Replication sync every 15 minutes (MLS Grid recommended) +*/15 * * * * cd /var/www/html && wp mls sync incremental --allow-root >> /var/log/mls-sync.log 2>&1 -# Media downloads every 5 minutes (processes up to 50 items per run) -*/5 * * * * cd /var/www/html && wp mls media process --limit=50 --quiet --allow-root >> /var/log/mls-media.log 2>&1 - -# Full sync weekly (Sunday 3am) -0 3 * * 0 cd /var/www/html && wp mls sync full --allow-root >> /var/log/mls-sync.log 2>&1 +# Full re-sync weekly (Sunday 3am) - rebuilds from scratch +0 3 * * 0 cd /var/www/html && wp mls cache clear --confirm --allow-root && wp mls sync full --allow-root >> /var/log/mls-sync.log 2>&1 ``` +Note: No separate media cron needed - images are fetched on-demand when properties are viewed. + ### Public API Functions Available for themes/plugins: @@ -178,9 +196,19 @@ $properties = mls_get_properties([ // Get single property $property = mls_get_property('NST123456'); -// Get media +// Get media (on-demand fetching) +$image_url = mls_get_property_image('NST123456'); // Fetches if not cached +$image_url = mls_get_property_image('NST123456', false); // Return null if not cached + +// Get all images (fetches first N on demand) +$images = mls_get_property_images('NST123456'); // Fetches first 1 if uncached +$images = mls_get_property_images('NST123456', 5); // Fetches first 5 if uncached + +// Get media metadata (no fetch) $media = mls_get_property_media('NST123456'); -$image_url = mls_get_property_image('NST123456'); + +// Get cache statistics +$stats = mls_get_cache_stats(); // Returns total_media, cached, uncached counts // Get distinct values $cities = mls_get_cities('Active'); @@ -189,20 +217,12 @@ $cities = mls_get_cities('Active'); if (mls_is_available()) { ... } ``` -### Sync Strategy - -1. **Property Sync**: Full/incremental sync downloads property data and queues media -2. **Media Queue**: Separate process downloads media with rate limiting -3. **Delete Handling**: MlgCanView=false triggers local deletion -4. **Media Storage**: Downloads to wp-content/uploads/mls-listings/ -5. **Recovery**: Stores last_next_link for resume on failure - ### Testing After Changes ```bash wp mls test connection wp mls test auth -wp mls sync full --dry-run --limit=10 +wp mls sync full --dry-run --limit=10 --verbose wp mls media status wp mls stats ``` @@ -226,3 +246,28 @@ Key fields from API to database: | MlgCanView | mlg_can_view | Full API response stored in `raw_data` column as JSON. + +## Troubleshooting + +### "Value out of range" error +The API is rejecting a high `$skip` value. This means pagination broke. Clear data and re-run initial sync: +```bash +wp mls cache clear --confirm --allow-root +wp mls sync full --allow-root +``` + +### All properties showing as "Sold" +The initial sync was run without the Active/Pending filter. Clear and re-sync: +```bash +wp mls cache clear --confirm --allow-root +wp mls sync full --allow-root +``` + +### Media not loading +Images are fetched on-demand. Check: +1. `wp mls media status` - see cache stats +2. `wp mls media fetch --listing=` - manually fetch for a listing +3. Check `wp-content/uploads/mls-listings/` directory permissions + +### Sync taking too long +Initial sync of ~30K Active/Pending properties takes about 30-45 minutes. Use `--verbose` to see progress. diff --git a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-api-client.php b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-api-client.php index e1cb4784..951f1197 100644 --- a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-api-client.php +++ b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-api-client.php @@ -318,26 +318,53 @@ class MLS_API_Client { } /** - * Get properties including those marked for deletion (for sync) + * Get properties for initial sync (Active/Pending only) * - * @param string|null $timestamp Optional modification timestamp filter * @param string|null $expand Expand parameter * @param int|null $top Number of records * @return array|WP_Error Response data or error */ - public function get_properties_for_sync($timestamp = null, $expand = null, $top = null) { - // Don't filter by MlgCanView for sync - we need to see deleted records + public function get_properties_for_initial_sync($expand = null, $top = null) { $params = array(); $system = $this->options->get_originating_system(); - if ($timestamp) { - $params['$filter'] = "OriginatingSystemName eq '{$system}' and ModificationTimestamp gt {$timestamp}"; - } else { - // Initial sync - only get viewable records - $params['$filter'] = "OriginatingSystemName eq '{$system}' and MlgCanView eq true"; + // Initial sync: only Active/Pending with MlgCanView=true + $params['$filter'] = "OriginatingSystemName eq '{$system}' and MlgCanView eq true and (StandardStatus eq 'Active' or StandardStatus eq 'Pending')"; + + if ($expand) { + $params['$expand'] = $expand; } + if ($top) { + $params['$top'] = min($top, $expand ? self::MAX_TOP_WITH_EXPAND : self::MAX_TOP_NO_EXPAND); + } else { + $params['$top'] = $expand ? self::MAX_TOP_WITH_EXPAND : self::DEFAULT_TOP; + } + + return $this->request('Property', $params); + } + + /** + * Get properties modified since timestamp (for replication) + * + * Does NOT filter by MlgCanView or StandardStatus so we can detect: + * - Records that became unavailable (MlgCanView=false) + * - Records that changed status (Active -> Sold) + * + * @param string $timestamp ISO 8601 modification timestamp + * @param string|null $expand Expand parameter + * @param int|null $top Number of records + * @return array|WP_Error Response data or error + */ + public function get_properties_for_replication($timestamp, $expand = null, $top = null) { + $params = array(); + + $system = $this->options->get_originating_system(); + + // Replication: get ALL changes since timestamp (no MlgCanView or Status filter) + $params['$filter'] = "OriginatingSystemName eq '{$system}' and ModificationTimestamp gt {$timestamp}"; + if ($expand) { $params['$expand'] = $expand; } diff --git a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php index 3ef93ed2..fd98adb0 100644 --- a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php +++ b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php @@ -2,11 +2,12 @@ /** * MLS Media Handler * - * Handles downloading and managing media files from MLS listings - * Uses a queue-based system with rate limiting to comply with API limits + * Handles on-demand fetching and caching of media files from MLS listings. + * Images are downloaded when first requested and cached locally. * - * Rate limits: 2 requests/second (500ms minimum between requests) - * We use 700ms between requests (25% buffer) + * Per MLS Grid rules: + * - MediaURLs must NOT be used directly on websites + * - Images must be downloaded and served from our own server */ if (!defined('ABSPATH')) { @@ -20,21 +21,6 @@ class MLS_Media_Handler { */ const UPLOAD_SUBDIR = 'mls-listings'; - /** - * Minimum delay between media downloads in milliseconds (700ms = 25% buffer over 500ms limit) - */ - const DOWNLOAD_DELAY_MS = 700; - - /** - * Retry backoff time in hours for failed downloads - */ - const RETRY_BACKOFF_HOURS = 3; - - /** - * Maximum download attempts before permanent failure - */ - const MAX_ATTEMPTS = 5; - /** * Database instance */ @@ -80,28 +66,28 @@ class MLS_Media_Handler { * @return string Absolute path */ public function get_listing_dir($listing_key) { - // Use first 2 characters as subdirectory to prevent too many files in one folder $prefix = substr($listing_key, 0, 2); return $this->get_upload_dir() . '/' . $prefix . '/' . $listing_key; } /** - * Queue media for a property (does NOT download immediately) + * Store media metadata from API sync (no download) * * @param string $listing_key Listing key * @param array $media_array Media array from API * @param callable|null $progress_callback Callback for progress updates + * @return array Stats */ - public function queue_property_media($listing_key, $media_array, $progress_callback = null) { + public function sync_property_media($listing_key, $media_array, $force = false, $progress_callback = null) { global $wpdb; if (empty($media_array)) { - return; + return array('stored' => 0, 'skipped' => 0); } $received_keys = array(); - $queued_count = 0; - $skipped_count = 0; + $stored = 0; + $skipped = 0; foreach ($media_array as $media) { $media_key = $media['MediaKey'] ?? null; @@ -134,40 +120,35 @@ class MLS_Media_Handler { ); if ($existing) { - // Update existing record + // Check if URL changed - if so, clear cached file + if ($existing->media_url !== ($media['MediaURL'] ?? null) && $existing->local_path) { + $file_path = $this->get_upload_dir() . '/' . $existing->local_path; + if (file_exists($file_path)) { + unlink($file_path); + } + $data['local_path'] = null; + $data['local_url'] = null; + $data['downloaded_at'] = null; + } + $wpdb->update( $this->db->media_table(), $data, array('id' => $existing->id) ); - - // Check if we need to re-download (queue it) - if ($this->needs_download($existing, $media)) { - $this->mark_for_download($existing->id); - $queued_count++; - if ($progress_callback) { - call_user_func($progress_callback, 'media_queued', array('media_key' => $media_key)); - } - } else { - $skipped_count++; - if ($progress_callback) { - call_user_func($progress_callback, 'media_skipped', array('media_key' => $media_key)); - } - } + $skipped++; } else { - // Insert new record - queued for download $data['created_at'] = current_time('mysql'); - $data['queued_at'] = current_time('mysql'); - $data['download_status'] = 'pending'; $wpdb->insert($this->db->media_table(), $data); - $queued_count++; - if ($progress_callback) { - call_user_func($progress_callback, 'media_queued', array('media_key' => $media_key)); - } + $stored++; + } + + if ($progress_callback) { + call_user_func($progress_callback, 'media_stored', array('media_key' => $media_key)); } } - // Delete media that no longer exists + // Delete orphaned media records if (!empty($received_keys)) { $placeholders = implode(',', array_fill(0, count($received_keys), '%s')); $values = array_merge(array($listing_key), $received_keys); @@ -179,347 +160,203 @@ class MLS_Media_Handler { )); foreach ($orphaned as $record) { - // Delete file if exists if ($record->local_path) { $file_path = $this->get_upload_dir() . '/' . $record->local_path; if (file_exists($file_path)) { unlink($file_path); } } - - // Delete record $wpdb->delete($this->db->media_table(), array('id' => $record->id)); } } - return array( - 'queued' => $queued_count, - 'skipped' => $skipped_count, - ); + return array('stored' => $stored, 'skipped' => $skipped); } /** - * Mark a media record for download + * Get image URL for a media record, fetching on-demand if needed * - * @param int $media_id Media ID + * @param int|object $media Media ID or media record object + * @param bool $fetch_if_missing Whether to fetch if not cached + * @return string|null Local URL or null */ - private function mark_for_download($media_id) { + public function get_image_url($media, $fetch_if_missing = true) { global $wpdb; - $wpdb->update( - $this->db->media_table(), - array( - 'download_status' => 'pending', - 'queued_at' => current_time('mysql'), - 'local_path' => null, - 'local_url' => null, - 'downloaded_at' => null, - 'download_error' => null, - ), - array('id' => $media_id) - ); + // Get media record if ID passed + if (is_numeric($media)) { + $media = $wpdb->get_row($wpdb->prepare( + "SELECT * FROM {$this->db->media_table()} WHERE id = %d", + $media + )); + } + + if (!$media) { + return null; + } + + // Already cached + if ($media->local_url && $media->local_path) { + $file_path = $this->get_upload_dir() . '/' . $media->local_path; + if (file_exists($file_path)) { + return $media->local_url; + } + } + + // Fetch on demand + if ($fetch_if_missing && $media->media_url) { + $result = $this->fetch_and_cache($media); + if ($result) { + return $result; + } + } + + return null; } /** - * Check if media needs to be downloaded + * Get primary image URL for a listing (on-demand) * - * @param object $existing Existing media record - * @param array $new_data New media data from API - * @return bool + * @param string $listing_key Listing key + * @param bool $fetch_if_missing Whether to fetch if not cached + * @return string|null Image URL */ - private function needs_download($existing, $new_data) { - // No local file - if (empty($existing->local_path)) { - return true; - } - - // File doesn't exist - $file_path = $this->get_upload_dir() . '/' . $existing->local_path; - if (!file_exists($file_path)) { - return true; - } - - // Media URL changed - if ($existing->media_url !== ($new_data['MediaURL'] ?? null)) { - return true; - } - - return false; - } - - /** - * Get the next media item to download from the queue - * - * @return object|null Media record or null if queue is empty - */ - public function get_next_queued() { + public function get_primary_image($listing_key, $fetch_if_missing = true) { global $wpdb; - $now = current_time('mysql'); - - // Get next pending item that's not in retry backoff - return $wpdb->get_row($wpdb->prepare( + // First check for already-cached image + $cached = $wpdb->get_row($wpdb->prepare( "SELECT * FROM {$this->db->media_table()} - WHERE download_status = 'pending' - AND media_url IS NOT NULL - AND download_attempts < %d - AND (retry_after IS NULL OR retry_after <= %s) - ORDER BY queued_at ASC + WHERE listing_key = %s AND local_url IS NOT NULL AND local_path IS NOT NULL + ORDER BY media_order ASC LIMIT 1", - self::MAX_ATTEMPTS, - $now + $listing_key )); - } - /** - * Get queue statistics - * - * @return array Queue stats - */ - public function get_queue_stats() { - global $wpdb; - - $now = current_time('mysql'); - - return array( - 'pending' => (int) $wpdb->get_var( - "SELECT COUNT(*) FROM {$this->db->media_table()} - WHERE download_status = 'pending'" - ), - 'ready' => (int) $wpdb->get_var($wpdb->prepare( - "SELECT COUNT(*) FROM {$this->db->media_table()} - WHERE download_status = 'pending' - AND media_url IS NOT NULL - AND download_attempts < %d - AND (retry_after IS NULL OR retry_after <= %s)", - self::MAX_ATTEMPTS, - $now - )), - 'in_backoff' => (int) $wpdb->get_var($wpdb->prepare( - "SELECT COUNT(*) FROM {$this->db->media_table()} - WHERE download_status = 'pending' - AND retry_after > %s", - $now - )), - 'failed' => (int) $wpdb->get_var($wpdb->prepare( - "SELECT COUNT(*) FROM {$this->db->media_table()} - WHERE download_status = 'failed' - OR download_attempts >= %d", - self::MAX_ATTEMPTS - )), - 'completed' => (int) $wpdb->get_var( - "SELECT COUNT(*) FROM {$this->db->media_table()} - WHERE download_status = 'completed'" - ), - ); - } - - /** - * Process media queue with rate limiting - * - * @param int $limit Max items to process - * @param callable|null $progress_callback Callback for progress updates - * @return array Processing stats - */ - public function process_queue($limit = 100, $progress_callback = null) { - $stats = array( - 'processed' => 0, - 'success' => 0, - 'failed' => 0, - 'skipped' => 0, - ); - - $last_download_time = 0; - - for ($i = 0; $i < $limit; $i++) { - $media = $this->get_next_queued(); - - if (!$media) { - // Queue empty - break; - } - - // Rate limiting: ensure minimum delay between downloads - $now_ms = microtime(true) * 1000; - $elapsed = $now_ms - $last_download_time; - - if ($elapsed < self::DOWNLOAD_DELAY_MS && $last_download_time > 0) { - $wait_ms = (int) (self::DOWNLOAD_DELAY_MS - $elapsed); - usleep($wait_ms * 1000); - } - - // Download the media - $result = $this->download_media($media->id); - $last_download_time = microtime(true) * 1000; - - $stats['processed']++; - - if ($result === true) { - $stats['success']++; - if ($progress_callback) { - call_user_func($progress_callback, 'media_downloaded', array( - 'media_key' => $media->media_key, - 'listing_key' => $media->listing_key, - )); - } - } elseif ($result === 'backoff') { - $stats['skipped']++; - if ($progress_callback) { - call_user_func($progress_callback, 'media_backoff', array( - 'media_key' => $media->media_key, - 'listing_key' => $media->listing_key, - )); - } - } else { - $stats['failed']++; - if ($progress_callback) { - call_user_func($progress_callback, 'media_error', array( - 'media_key' => $media->media_key, - 'listing_key' => $media->listing_key, - 'error' => $result, - )); - } + if ($cached) { + $file_path = $this->get_upload_dir() . '/' . $cached->local_path; + if (file_exists($file_path)) { + return $cached->local_url; } } - return $stats; - } - - /** - * Download a media file - * - * @param int $media_id Media record ID - * @return bool|string True on success, 'backoff' if set for retry, error message on failure - */ - public function download_media($media_id) { - global $wpdb; - + // Get first media record (may not be cached) $media = $wpdb->get_row($wpdb->prepare( - "SELECT * FROM {$this->db->media_table()} WHERE id = %d", - $media_id + "SELECT * FROM {$this->db->media_table()} + WHERE listing_key = %s AND media_url IS NOT NULL + ORDER BY media_order ASC + LIMIT 1", + $listing_key )); - if (!$media || empty($media->media_url)) { - return 'No media URL'; + if (!$media) { + return null; } - // Increment attempt counter - $wpdb->update( - $this->db->media_table(), - array('download_attempts' => $media->download_attempts + 1), - array('id' => $media_id) - ); + // If already cached and file exists, return it + if ($media->local_url && $media->local_path) { + $file_path = $this->get_upload_dir() . '/' . $media->local_path; + if (file_exists($file_path)) { + return $media->local_url; + } + } - // Make the request - $start_time = microtime(true); + // Fetch on demand + if ($fetch_if_missing) { + return $this->fetch_and_cache($media); + } - $response = wp_remote_get($media->media_url, array( - 'timeout' => 60, - 'stream' => false, + return null; + } + + /** + * Get all images for a listing (on-demand for first N) + * + * @param string $listing_key Listing key + * @param int $fetch_limit Max images to fetch on-demand (0 = none) + * @return array Media records with local_url populated where available + */ + public function get_listing_images($listing_key, $fetch_limit = 1) { + global $wpdb; + + $media = $wpdb->get_results($wpdb->prepare( + "SELECT * FROM {$this->db->media_table()} + WHERE listing_key = %s + ORDER BY media_order ASC", + $listing_key )); - $response_time_ms = (int) ((microtime(true) - $start_time) * 1000); - $status_code = 0; - $error_msg = null; + if (empty($media)) { + return array(); + } + + $fetched = 0; + foreach ($media as &$item) { + // Check if cached and file exists + if ($item->local_url && $item->local_path) { + $file_path = $this->get_upload_dir() . '/' . $item->local_path; + if (file_exists($file_path)) { + continue; + } + } + + // Fetch on demand up to limit + if ($fetched < $fetch_limit && $item->media_url) { + $url = $this->fetch_and_cache($item); + if ($url) { + $item->local_url = $url; + $fetched++; + } + } + } + + return $media; + } + + /** + * Fetch image from MLS Grid and cache locally + * + * @param object $media Media record + * @return string|null Local URL on success, null on failure + */ + private function fetch_and_cache($media) { + global $wpdb; + + if (empty($media->media_url)) { + return null; + } + + // Download the image + $response = wp_remote_get($media->media_url, array( + 'timeout' => 30, + )); if (is_wp_error($response)) { - $error_msg = $response->get_error_message(); - $this->log_download($media, 'error', null, $response_time_ms, $error_msg); - $this->handle_download_failure($media_id, $error_msg, false); - return $error_msg; + $this->logger->warning('Media fetch failed', array( + 'listing_key' => $media->listing_key, + 'media_key' => $media->media_key, + 'error' => $response->get_error_message(), + )); + return null; } $status_code = wp_remote_retrieve_response_code($response); - $this->log_download($media, 'attempt', $status_code, $response_time_ms, null); - // Success - if ($status_code === 200) { - $body = wp_remote_retrieve_body($response); - if (empty($body)) { - $error_msg = 'Empty response body'; - $this->log_download($media, 'error', $status_code, $response_time_ms, $error_msg); - $this->handle_download_failure($media_id, $error_msg, false); - return $error_msg; - } - - // Save the file - $save_result = $this->save_media_file($media, $body, $response); - if ($save_result !== true) { - $this->log_download($media, 'error', $status_code, $response_time_ms, $save_result); - $this->handle_download_failure($media_id, $save_result, false); - return $save_result; - } - - $this->log_download($media, 'success', $status_code, $response_time_ms, null); - return true; + if ($status_code !== 200) { + $this->logger->warning('Media fetch HTTP error', array( + 'listing_key' => $media->listing_key, + 'media_key' => $media->media_key, + 'status' => $status_code, + )); + return null; } - // Rate limited (429) or server error (5xx) - set backoff - $retryable = in_array($status_code, array(429, 500, 502, 503, 504)); - $error_msg = "HTTP {$status_code}"; - - if ($retryable) { - $this->log_download($media, 'rate_limited', $status_code, $response_time_ms, $error_msg); - $this->handle_download_failure($media_id, $error_msg, true); - return 'backoff'; + $body = wp_remote_retrieve_body($response); + if (empty($body)) { + return null; } - // Permanent failure (404, 403, etc.) - $this->log_download($media, 'permanent_error', $status_code, $response_time_ms, $error_msg); - $this->handle_download_failure($media_id, $error_msg, false); - return $error_msg; - } - - /** - * Handle download failure - * - * @param int $media_id Media ID - * @param string $error Error message - * @param bool $set_backoff Whether to set retry backoff - */ - private function handle_download_failure($media_id, $error, $set_backoff) { - global $wpdb; - - $media = $wpdb->get_row($wpdb->prepare( - "SELECT * FROM {$this->db->media_table()} WHERE id = %d", - $media_id - )); - - $update_data = array( - 'download_error' => $error, - ); - - if ($set_backoff) { - // Set retry_after to 3 hours from now - $retry_after = date('Y-m-d H:i:s', strtotime('+' . self::RETRY_BACKOFF_HOURS . ' hours')); - $update_data['retry_after'] = $retry_after; - } - - // Check if max attempts reached - if ($media && $media->download_attempts >= self::MAX_ATTEMPTS) { - $update_data['download_status'] = 'failed'; - $this->log_missing_media($media, $error); - } - - $wpdb->update( - $this->db->media_table(), - $update_data, - array('id' => $media_id) - ); - } - - /** - * Save downloaded media file to disk - * - * @param object $media Media record - * @param string $body File contents - * @param array $response HTTP response - * @return bool|string True on success, error message on failure - */ - private function save_media_file($media, $body, $response) { - global $wpdb; - - // Determine file extension from content type or URL + // Determine extension $content_type = wp_remote_retrieve_header($response, 'content-type'); $extension = $this->get_extension_from_content_type($content_type, $media->media_url); @@ -534,15 +371,17 @@ class MLS_Media_Handler { $file_path = $listing_dir . '/' . $filename; if (file_put_contents($file_path, $body) === false) { - return 'Failed to write file'; + $this->logger->error('Failed to write media file', array( + 'path' => $file_path, + )); + return null; } - // Calculate relative path + // Update database $prefix = substr($media->listing_key, 0, 2); $relative_path = $prefix . '/' . $media->listing_key . '/' . $filename; $local_url = $this->get_upload_url() . '/' . $relative_path; - // Update record $wpdb->update( $this->db->media_table(), array( @@ -551,42 +390,17 @@ class MLS_Media_Handler { 'file_size' => strlen($body), 'mime_type' => $content_type, 'downloaded_at' => current_time('mysql'), - 'download_error' => null, - 'download_status' => 'completed', - 'retry_after' => null, ), array('id' => $media->id) ); - return true; - } + $this->logger->debug('Media fetched and cached', array( + 'listing_key' => $media->listing_key, + 'media_key' => $media->media_key, + 'size' => strlen($body), + )); - /** - * Log a download attempt to the media log table - * - * @param object $media Media record - * @param string $action Action type (attempt, success, error, rate_limited, permanent_error) - * @param int|null $status_code HTTP status code - * @param int $response_time_ms Response time in milliseconds - * @param string|null $error Error message - */ - private function log_download($media, $action, $status_code, $response_time_ms, $error) { - global $wpdb; - - $wpdb->insert( - $this->db->media_log_table(), - array( - 'media_id' => $media->id, - 'listing_key' => $media->listing_key, - 'media_key' => $media->media_key, - 'action' => $action, - 'status_code' => $status_code, - 'response_time_ms' => $response_time_ms, - 'error_message' => $error, - 'url' => $media->media_url, - 'created_at' => current_time('mysql'), - ) - ); + return $local_url; } /** @@ -597,7 +411,6 @@ class MLS_Media_Handler { * @return string File extension */ private function get_extension_from_content_type($content_type, $url) { - // Extract main type from content-type header $content_type = strtolower(explode(';', $content_type)[0]); $map = array( @@ -619,60 +432,6 @@ class MLS_Media_Handler { return $ext ?: 'jpg'; } - /** - * Get the path to the missing media log file - * - * @return string File path - */ - public function get_missing_log_path() { - $upload_dir = wp_upload_dir(); - return $upload_dir['basedir'] . '/mls-missing-media.log'; - } - - /** - * Log a failed media download to the missing media log file - * - * @param object $media Media record - * @param string $error Error message - */ - private function log_missing_media($media, $error) { - $log_file = $this->get_missing_log_path(); - $timestamp = date('Y-m-d H:i:s'); - $line = sprintf( - "[%s] %s | %s | %s | %s\n", - $timestamp, - $media->listing_key, - $media->media_key, - $error, - $media->media_url - ); - file_put_contents($log_file, $line, FILE_APPEND | LOCK_EX); - } - - /** - * Clear the missing media log file - */ - public function clear_missing_log() { - $log_file = $this->get_missing_log_path(); - if (file_exists($log_file)) { - unlink($log_file); - } - } - - /** - * Get missing media count from log file - * - * @return int Number of missing media entries - */ - public function get_missing_count() { - $log_file = $this->get_missing_log_path(); - if (!file_exists($log_file)) { - return 0; - } - $content = file_get_contents($log_file); - return substr_count($content, "\n"); - } - /** * Delete all media for a property * @@ -719,7 +478,7 @@ class MLS_Media_Handler { } /** - * Get media for a listing + * Get media for a listing (legacy compatibility) * * @param string $listing_key Listing key * @return array Media records @@ -736,80 +495,13 @@ class MLS_Media_Handler { } /** - * Get primary image URL for a listing + * Clean up orphaned media files (files without database records) * - * @param string $listing_key Listing key - * @return string|null Image URL - */ - public function get_primary_image($listing_key) { - global $wpdb; - - $media = $wpdb->get_row($wpdb->prepare( - "SELECT local_url, media_url FROM {$this->db->media_table()} - WHERE listing_key = %s AND local_path IS NOT NULL - ORDER BY media_order ASC - LIMIT 1", - $listing_key - )); - - if ($media && $media->local_url) { - return $media->local_url; - } - - return null; - } - - /** - * Reset failed downloads for retry - * - * @param string|null $listing_key Optional listing key to filter - * @return int Number of records reset - */ - public function reset_failed_downloads($listing_key = null) { - global $wpdb; - - $where = "download_status = 'failed' OR download_attempts >= " . self::MAX_ATTEMPTS; - $values = array(); - - if ($listing_key) { - $where .= " AND listing_key = %s"; - $values[] = $listing_key; - } - - if (!empty($values)) { - $sql = $wpdb->prepare( - "UPDATE {$this->db->media_table()} - SET download_status = 'pending', - download_attempts = 0, - download_error = NULL, - retry_after = NULL, - queued_at = %s - WHERE {$where}", - array_merge(array(current_time('mysql')), $values) - ); - } else { - $sql = $wpdb->prepare( - "UPDATE {$this->db->media_table()} - SET download_status = 'pending', - download_attempts = 0, - download_error = NULL, - retry_after = NULL, - queued_at = %s - WHERE {$where}", - current_time('mysql') - ); - } - - $wpdb->query($sql); - return $wpdb->rows_affected; - } - - /** - * Clean up orphaned media (files without database records) - * - * @return int Number of files deleted + * @return int Number of directories deleted */ public function cleanup_orphaned_files() { + global $wpdb; + $deleted = 0; $base_dir = $this->get_upload_dir(); @@ -817,7 +509,6 @@ class MLS_Media_Handler { return 0; } - // Iterate through prefix directories foreach (scandir($base_dir) as $prefix) { if ($prefix === '.' || $prefix === '..' || !is_dir($base_dir . '/' . $prefix)) { continue; @@ -825,7 +516,6 @@ class MLS_Media_Handler { $prefix_dir = $base_dir . '/' . $prefix; - // Iterate through listing directories foreach (scandir($prefix_dir) as $listing_key) { if ($listing_key === '.' || $listing_key === '..') { continue; @@ -836,8 +526,6 @@ class MLS_Media_Handler { continue; } - // Check if listing exists in database - global $wpdb; $exists = $wpdb->get_var($wpdb->prepare( "SELECT COUNT(*) FROM {$this->db->properties_table()} WHERE listing_key = %s", $listing_key @@ -854,76 +542,57 @@ class MLS_Media_Handler { } /** - * Get recent download logs + * Get cache statistics * - * @param int $limit Number of entries to return - * @param string|null $action Optional action filter - * @return array Log entries + * @return array Cache stats */ - public function get_download_logs($limit = 100, $action = null) { + public function get_cache_stats() { global $wpdb; - $where = ''; - $values = array(); + return array( + 'total_media' => (int) $wpdb->get_var( + "SELECT COUNT(*) FROM {$this->db->media_table()}" + ), + 'cached' => (int) $wpdb->get_var( + "SELECT COUNT(*) FROM {$this->db->media_table()} WHERE local_url IS NOT NULL" + ), + 'uncached' => (int) $wpdb->get_var( + "SELECT COUNT(*) FROM {$this->db->media_table()} WHERE local_url IS NULL" + ), + ); + } - if ($action) { - $where = "WHERE action = %s"; - $values[] = $action; + /** + * Get path to missing media log file (legacy compatibility) + * + * @return string File path + */ + public function get_missing_log_path() { + $upload_dir = wp_upload_dir(); + return $upload_dir['basedir'] . '/mls-missing-media.log'; + } + + /** + * Get missing media count (legacy compatibility) + * + * @return int + */ + public function get_missing_count() { + $log_file = $this->get_missing_log_path(); + if (!file_exists($log_file)) { + return 0; } - - $values[] = $limit; - - return $wpdb->get_results($wpdb->prepare( - "SELECT * FROM {$this->db->media_log_table()} - {$where} - ORDER BY created_at DESC - LIMIT %d", - $values - )); + $content = file_get_contents($log_file); + return substr_count($content, "\n"); } /** - * Clear old download logs - * - * @param int $days_old Delete logs older than this many days - * @return int Number of entries deleted + * Clear missing log (legacy compatibility) */ - public function clear_old_logs($days_old = 7) { - global $wpdb; - - $cutoff = date('Y-m-d H:i:s', strtotime("-{$days_old} days")); - - $wpdb->query($wpdb->prepare( - "DELETE FROM {$this->db->media_log_table()} WHERE created_at < %s", - $cutoff - )); - - return $wpdb->rows_affected; - } - - /** - * Legacy sync method - now queues media instead of downloading immediately - * Kept for backward compatibility - * - * @param string $listing_key Listing key - * @param array $media_array Media array from API - * @param bool $force Force re-download all media - * @param callable|null $progress_callback Callback for progress updates - */ - public function sync_property_media($listing_key, $media_array, $force = false, $progress_callback = null) { - // Now just queues media - actual download happens via process_queue() - return $this->queue_property_media($listing_key, $media_array, $progress_callback); - } - - /** - * Legacy download_pending method - now uses process_queue - * Kept for backward compatibility - * - * @param int $limit Max media to download - * @param callable|null $progress_callback Callback for progress updates - * @return array Stats - */ - public function download_pending($limit = 100, $progress_callback = null) { - return $this->process_queue($limit, $progress_callback); + public function clear_missing_log() { + $log_file = $this->get_missing_log_path(); + if (file_exists($log_file)) { + unlink($log_file); + } } } diff --git a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php index df60eabc..1fc4ef61 100644 --- a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php +++ b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php @@ -78,7 +78,10 @@ class MLS_Sync_Engine { } /** - * Run full sync + * Run full sync (Active/Pending properties only) + * + * Initial import fetches only Active and Pending properties. + * Use incremental sync (replication) for ongoing updates. * * @param bool $dry_run If true, don't make changes * @param int|null $limit Max records to process @@ -86,7 +89,7 @@ class MLS_Sync_Engine { * @return array Sync results */ public function run_full_sync($dry_run = false, $limit = null, $progress_callback = null) { - $this->logger->info('Starting full sync', array('dry_run' => $dry_run, 'limit' => $limit)); + $this->logger->info('Starting full sync (Active/Pending only)', array('dry_run' => $dry_run, 'limit' => $limit)); // Store progress callback for use in process_property $this->progress_callback = $progress_callback; @@ -106,15 +109,15 @@ class MLS_Sync_Engine { ); try { - // Get first page of properties with media + // Get first page of Active/Pending properties with media $start_time = microtime(true); $this->emit_progress('api_request', array( 'method' => 'GET', 'url' => 'Property', - 'params' => array('type' => 'full_sync', 'limit' => $limit), + 'params' => array('type' => 'initial_sync', 'filter' => 'Active/Pending', 'limit' => $limit), )); - $response = $this->api_client->get_properties_for_sync(null, 'Media', $limit ? min($limit, 1000) : null); + $response = $this->api_client->get_properties_for_initial_sync('Media', $limit ? min($limit, 1000) : null); $elapsed = round((microtime(true) - $start_time) * 1000); if (is_wp_error($response)) { @@ -239,7 +242,13 @@ class MLS_Sync_Engine { } /** - * Run incremental sync + * Run incremental sync (replication) + * + * Fetches all properties modified since last sync, including those that: + * - Became unavailable (MlgCanView=false) + * - Changed status (Active -> Sold) + * + * Properties are deleted from local DB if MlgCanView=false or status not Active/Pending. * * @param bool $dry_run If true, don't make changes * @param callable|null $progress_callback Callback for progress updates @@ -254,7 +263,7 @@ class MLS_Sync_Engine { return $this->run_full_sync($dry_run, null, $progress_callback); } - $this->logger->info('Starting incremental sync', array( + $this->logger->info('Starting replication sync', array( 'since' => $last_timestamp, 'dry_run' => $dry_run, )); @@ -276,15 +285,15 @@ class MLS_Sync_Engine { ); try { - // Get modified properties (including those marked for deletion) + // Get ALL modified properties (no MlgCanView or status filter for replication) $start_time = microtime(true); $this->emit_progress('api_request', array( 'method' => 'GET', 'url' => 'Property', - 'params' => array('type' => 'incremental', 'since' => $last_timestamp), + 'params' => array('type' => 'replication', 'since' => $last_timestamp), )); - $response = $this->api_client->get_properties_since($last_timestamp, 'Media'); + $response = $this->api_client->get_properties_for_replication($last_timestamp, 'Media'); $elapsed = round((microtime(true) - $start_time) * 1000); if (is_wp_error($response)) { @@ -525,9 +534,18 @@ class MLS_Sync_Engine { */ private $progress_callback = null; + /** + * Allowed statuses for our database (Active/Pending only) + */ + const ALLOWED_STATUSES = array('Active', 'Pending'); + /** * Process a single property record * + * During replication, properties are deleted if: + * - MlgCanView = false (removed from feed) + * - StandardStatus not in (Active, Pending) + * * @param array $property Property data from API * @param bool $dry_run If true, don't make changes */ @@ -543,15 +561,31 @@ class MLS_Sync_Engine { return; } - // Check MlgCanView - if false, delete the record + // Check MlgCanView and StandardStatus $can_view = $property['MlgCanView'] ?? true; + $status = $property['StandardStatus'] ?? null; - if (!$can_view) { - if (!$dry_run) { - $this->delete_property($listing_key); + // Delete if: not viewable OR status is not Active/Pending + $should_delete = !$can_view || !in_array($status, self::ALLOWED_STATUSES); + + if ($should_delete) { + // Check if we have this record locally before attempting delete + $exists_locally = $wpdb->get_var($wpdb->prepare( + "SELECT id FROM {$this->db->properties_table()} WHERE listing_key = %s", + $listing_key + )); + + if ($exists_locally) { + if (!$dry_run) { + $this->delete_property($listing_key); + } + $this->stats['deleted']++; + $this->emit_progress('property_deleted', array( + 'listing_key' => $listing_key, + 'reason' => !$can_view ? 'MlgCanView=false' : "Status={$status}", + )); } - $this->stats['deleted']++; - $this->emit_progress('property_deleted', array('listing_key' => $listing_key)); + // If not in our DB, just skip silently (e.g., Sold property we never had) return; } diff --git a/wp-content/plugins/mls-by-hansonxyz/mls-by-hansonxyz.php b/wp-content/plugins/mls-by-hansonxyz/mls-by-hansonxyz.php index ad4845ff..7dacaeb0 100644 --- a/wp-content/plugins/mls-by-hansonxyz/mls-by-hansonxyz.php +++ b/wp-content/plugins/mls-by-hansonxyz/mls-by-hansonxyz.php @@ -271,17 +271,21 @@ function mls_get_property_media($listing_key) { } /** - * Get primary image URL for a listing + * Get primary image URL for a listing (on-demand fetching) + * + * Images are fetched from MLS Grid and cached locally on first request. + * Per MLS Grid rules, images must be served from our own server. * * @param string $listing_key The listing key + * @param bool $fetch_if_missing Whether to fetch from MLS Grid if not cached (default: true) * @return string|null Image URL or null */ -function mls_get_property_image($listing_key) { +function mls_get_property_image($listing_key, $fetch_if_missing = true) { $plugin = mls_plugin(); - if (!$plugin->get_query()) { + if (!$plugin->get_media_handler()) { return null; } - return $plugin->get_query()->get_primary_image($listing_key); + return $plugin->get_media_handler()->get_primary_image($listing_key, $fetch_if_missing); } /** @@ -324,3 +328,34 @@ function mls_get_property_count($args = array()) { } return $plugin->get_query()->get_count($args); } + +/** + * Get all images for a listing (on-demand fetching) + * + * Returns all media records with local_url populated where cached. + * Can optionally fetch first N uncached images on-demand. + * + * @param string $listing_key The listing key + * @param int $fetch_limit Max images to fetch on-demand (default: 1, 0 = none) + * @return array Array of media objects + */ +function mls_get_property_images($listing_key, $fetch_limit = 1) { + $plugin = mls_plugin(); + if (!$plugin->get_media_handler()) { + return array(); + } + return $plugin->get_media_handler()->get_listing_images($listing_key, $fetch_limit); +} + +/** + * Get media cache statistics + * + * @return array Stats with total_media, cached, uncached counts + */ +function mls_get_cache_stats() { + $plugin = mls_plugin(); + if (!$plugin->get_media_handler()) { + return array('total_media' => 0, 'cached' => 0, 'uncached' => 0); + } + return $plugin->get_media_handler()->get_cache_stats(); +}