Refactor MLS sync to Active/Pending only with on-demand media
Major changes to sync strategy following MLS Grid best practices: - Initial sync now fetches only Active/Pending properties (~30K vs 1.3M) - Replication (incremental) fetches all changes, deletes non-Active/Pending - On-demand media fetching replaces background queue (avoids rate limits) - Media downloaded and cached when first viewed, not during sync - Updated CLI commands: wp mls media status/fetch/clear - Comprehensive documentation with troubleshooting guide This fixes the "Value out of range" API error caused by high $skip values. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Hanson.xyz Dev
@@ -338,14 +338,12 @@ class MLS_CLI {
|
||||
break;
|
||||
|
||||
case 'media':
|
||||
// Redirect to the new media command
|
||||
WP_CLI::line('Note: "wp mls sync media" is deprecated. Use "wp mls media process" instead.');
|
||||
// Media is now on-demand, this sync type is deprecated
|
||||
WP_CLI::line('Note: "wp mls sync media" is deprecated.');
|
||||
WP_CLI::line('Media is now fetched on-demand when properties are viewed on the website.');
|
||||
WP_CLI::line('');
|
||||
$this->media(array('process'), array(
|
||||
'limit' => $limit ?: 100,
|
||||
'verbose' => $verbose,
|
||||
'quiet' => $quiet,
|
||||
));
|
||||
WP_CLI::line('Use "wp mls media status" to see cache statistics.');
|
||||
WP_CLI::line('Use "wp mls media fetch --listing=<key>" to pre-cache a specific listing.');
|
||||
break;
|
||||
|
||||
case 'resume':
|
||||
@@ -799,263 +797,119 @@ class MLS_CLI {
|
||||
}
|
||||
|
||||
/**
|
||||
* Manage media download queue.
|
||||
* Show media cache status and manage cached files.
|
||||
*
|
||||
* Media is now fetched on-demand when properties are viewed on the website.
|
||||
* This command shows cache statistics and allows management of cached files.
|
||||
*
|
||||
* ## OPTIONS
|
||||
*
|
||||
* <action>
|
||||
* : Action: queue, process, status, reset, logs
|
||||
* [<action>]
|
||||
* : Action: status (default), fetch, clear
|
||||
*
|
||||
* [--listing=<key>]
|
||||
* : Listing key for fetch or clear actions
|
||||
*
|
||||
* [--limit=<n>]
|
||||
* : Limit number of items to process
|
||||
*
|
||||
* [--verbose]
|
||||
* : Show detailed output
|
||||
*
|
||||
* [--quiet]
|
||||
* : Suppress progress output
|
||||
*
|
||||
* [--days=<n>]
|
||||
* : Days of logs to keep (for logs --clear)
|
||||
*
|
||||
* [--clear]
|
||||
* : Clear logs older than --days
|
||||
* : For fetch action, max images to fetch (default: 1)
|
||||
*
|
||||
* ## EXAMPLES
|
||||
*
|
||||
* wp mls media status # Show queue statistics
|
||||
* wp mls media process # Process pending downloads (rate limited)
|
||||
* wp mls media process --limit=50 # Process up to 50 items
|
||||
* wp mls media reset # Reset failed downloads for retry
|
||||
* wp mls media logs # Show recent download logs
|
||||
* wp mls media logs --clear --days=7 # Clear logs older than 7 days
|
||||
* wp mls media status # Show cache statistics
|
||||
* wp mls media fetch --listing=NST123456 # Fetch images for a listing
|
||||
* wp mls media fetch --listing=NST123456 --limit=10 # Fetch up to 10 images
|
||||
* wp mls media clear --listing=NST123456 # Clear cached images for a listing
|
||||
*
|
||||
* @subcommand media
|
||||
*/
|
||||
public function media($args, $assoc_args) {
|
||||
$action = isset($args[0]) ? $args[0] : 'status';
|
||||
$limit = isset($assoc_args['limit']) ? (int) $assoc_args['limit'] : 100;
|
||||
$verbose = isset($assoc_args['verbose']);
|
||||
$quiet = isset($assoc_args['quiet']);
|
||||
|
||||
$media_handler = $this->plugin->get_media_handler();
|
||||
|
||||
switch ($action) {
|
||||
case 'status':
|
||||
case 'queue':
|
||||
$stats = $media_handler->get_queue_stats();
|
||||
$stats = $media_handler->get_cache_stats();
|
||||
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line('=== Media Download Queue ===');
|
||||
WP_CLI::line('=== Media Cache Status ===');
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line(sprintf('Pending total: %d', $stats['pending']));
|
||||
WP_CLI::line(sprintf('Ready now: %d', $stats['ready']));
|
||||
WP_CLI::line(sprintf('In backoff: %d (retry scheduled)', $stats['in_backoff']));
|
||||
WP_CLI::line(sprintf('Failed: %d (max attempts reached)', $stats['failed']));
|
||||
WP_CLI::line(sprintf('Completed: %d', $stats['completed']));
|
||||
WP_CLI::line(sprintf('Total media records: %d', $stats['total_media']));
|
||||
WP_CLI::line(sprintf('Cached locally: %d', $stats['cached']));
|
||||
WP_CLI::line(sprintf('Not yet cached: %d', $stats['uncached']));
|
||||
WP_CLI::line('');
|
||||
|
||||
if ($stats['ready'] > 0) {
|
||||
WP_CLI::line(sprintf(
|
||||
'Run "wp mls media process --limit=%d" to download pending media.',
|
||||
min($stats['ready'], 100)
|
||||
));
|
||||
WP_CLI::line(sprintf(
|
||||
'Estimated time: %d minutes (at 700ms per image)',
|
||||
ceil($stats['ready'] * 0.7 / 60)
|
||||
));
|
||||
}
|
||||
|
||||
if ($stats['failed'] > 0) {
|
||||
$cache_percent = $stats['total_media'] > 0
|
||||
? round(($stats['cached'] / $stats['total_media']) * 100, 1)
|
||||
: 0;
|
||||
WP_CLI::line(sprintf('Cache rate: %.1f%%', $cache_percent));
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line('Run "wp mls media reset" to retry failed downloads.');
|
||||
}
|
||||
WP_CLI::line('Images are fetched on-demand when properties are viewed.');
|
||||
WP_CLI::line('Use "wp mls media fetch --listing=<key>" to pre-cache specific listings.');
|
||||
WP_CLI::line('');
|
||||
break;
|
||||
|
||||
case 'process':
|
||||
$stats = $media_handler->get_queue_stats();
|
||||
|
||||
if ($stats['ready'] === 0) {
|
||||
WP_CLI::success('No media ready to download.');
|
||||
break;
|
||||
case 'fetch':
|
||||
$listing_key = isset($assoc_args['listing']) ? $assoc_args['listing'] : null;
|
||||
if (!$listing_key) {
|
||||
WP_CLI::error('Please specify --listing=<key>');
|
||||
}
|
||||
|
||||
$process_count = min($limit, $stats['ready']);
|
||||
$limit = isset($assoc_args['limit']) ? (int) $assoc_args['limit'] : 1;
|
||||
|
||||
WP_CLI::line(sprintf(
|
||||
'Processing %d media items (rate limited: 1 per 700ms)...',
|
||||
$process_count
|
||||
));
|
||||
WP_CLI::line(sprintf(
|
||||
'Estimated time: %d minutes',
|
||||
ceil($process_count * 0.7 / 60)
|
||||
));
|
||||
WP_CLI::line(sprintf('Fetching up to %d images for listing %s...', $limit, $listing_key));
|
||||
|
||||
if (!$quiet) {
|
||||
WP_CLI::line('Legend: P=downloaded B=backoff (retry later) E=error');
|
||||
echo "\n";
|
||||
$images = $media_handler->get_listing_images($listing_key, $limit);
|
||||
|
||||
$cached_count = 0;
|
||||
foreach ($images as $img) {
|
||||
if ($img->local_url) {
|
||||
$cached_count++;
|
||||
}
|
||||
}
|
||||
|
||||
// Progress callback
|
||||
$progress_callback = null;
|
||||
if (!$quiet) {
|
||||
$progress_callback = function($event, $data = array()) use ($verbose) {
|
||||
if ($verbose) {
|
||||
$this->output_verbose_media_event($event, $data);
|
||||
WP_CLI::line(sprintf(
|
||||
'Result: %d/%d images now cached for this listing.',
|
||||
$cached_count,
|
||||
count($images)
|
||||
));
|
||||
|
||||
if ($cached_count > 0) {
|
||||
WP_CLI::success('Images fetched successfully.');
|
||||
} elseif (count($images) === 0) {
|
||||
WP_CLI::warning('No media records found for this listing.');
|
||||
} else {
|
||||
switch ($event) {
|
||||
case 'media_downloaded':
|
||||
echo 'P';
|
||||
break;
|
||||
case 'media_backoff':
|
||||
echo 'B';
|
||||
break;
|
||||
case 'media_error':
|
||||
echo 'E';
|
||||
break;
|
||||
WP_CLI::warning('Failed to fetch images. Check logs for details.');
|
||||
}
|
||||
}
|
||||
};
|
||||
break;
|
||||
|
||||
case 'clear':
|
||||
$listing_key = isset($assoc_args['listing']) ? $assoc_args['listing'] : null;
|
||||
if (!$listing_key) {
|
||||
WP_CLI::error('Please specify --listing=<key>. To clear all media, use "wp mls cache clear --confirm".');
|
||||
}
|
||||
|
||||
$result = $media_handler->process_queue($process_count, $progress_callback);
|
||||
// Just clear the local files, keep metadata
|
||||
global $wpdb;
|
||||
$listing_dir = $media_handler->get_listing_dir($listing_key);
|
||||
|
||||
if (!$quiet) {
|
||||
echo "\n\n";
|
||||
if (is_dir($listing_dir)) {
|
||||
$this->recursive_delete($listing_dir);
|
||||
}
|
||||
|
||||
WP_CLI::line(sprintf(
|
||||
'Results: %d success, %d backoff, %d failed out of %d processed',
|
||||
$result['success'],
|
||||
$result['skipped'],
|
||||
$result['failed'],
|
||||
$result['processed']
|
||||
// Clear local_path and local_url but keep the records
|
||||
$wpdb->query($wpdb->prepare(
|
||||
"UPDATE {$this->plugin->get_db()->media_table()}
|
||||
SET local_path = NULL, local_url = NULL, downloaded_at = NULL
|
||||
WHERE listing_key = %s",
|
||||
$listing_key
|
||||
));
|
||||
|
||||
// Show updated stats
|
||||
$new_stats = $media_handler->get_queue_stats();
|
||||
WP_CLI::line(sprintf('Queue remaining: %d ready, %d in backoff', $new_stats['ready'], $new_stats['in_backoff']));
|
||||
|
||||
if ($result['failed'] > 0 || $result['skipped'] > 0) {
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line('Items in backoff will be retried after 3 hours.');
|
||||
WP_CLI::line('Run "wp mls media logs" to see download history.');
|
||||
}
|
||||
|
||||
if ($result['success'] > 0) {
|
||||
WP_CLI::success('Media processing complete.');
|
||||
}
|
||||
break;
|
||||
|
||||
case 'reset':
|
||||
WP_CLI::line('Resetting failed downloads for retry...');
|
||||
|
||||
$reset_count = $media_handler->reset_failed_downloads();
|
||||
|
||||
if ($reset_count > 0) {
|
||||
WP_CLI::success(sprintf('Reset %d failed downloads. They will be retried on next process.', $reset_count));
|
||||
} else {
|
||||
WP_CLI::success('No failed downloads to reset.');
|
||||
}
|
||||
break;
|
||||
|
||||
case 'logs':
|
||||
if (isset($assoc_args['clear'])) {
|
||||
$days = isset($assoc_args['days']) ? (int) $assoc_args['days'] : 7;
|
||||
$deleted = $media_handler->clear_old_logs($days);
|
||||
WP_CLI::success(sprintf('Deleted %d log entries older than %d days.', $deleted, $days));
|
||||
break;
|
||||
}
|
||||
|
||||
$logs = $media_handler->get_download_logs($limit);
|
||||
|
||||
if (empty($logs)) {
|
||||
WP_CLI::success('No download logs found.');
|
||||
break;
|
||||
}
|
||||
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line('=== Recent Download Logs ===');
|
||||
WP_CLI::line('');
|
||||
|
||||
foreach ($logs as $log) {
|
||||
$status_indicator = '';
|
||||
switch ($log->action) {
|
||||
case 'success':
|
||||
$status_indicator = '[OK]';
|
||||
break;
|
||||
case 'rate_limited':
|
||||
$status_indicator = '[429]';
|
||||
break;
|
||||
case 'permanent_error':
|
||||
$status_indicator = '[ERR]';
|
||||
break;
|
||||
case 'error':
|
||||
$status_indicator = '[FAIL]';
|
||||
break;
|
||||
default:
|
||||
$status_indicator = "[{$log->action}]";
|
||||
}
|
||||
|
||||
$line = sprintf(
|
||||
'%s %s %s %s %dms',
|
||||
$log->created_at,
|
||||
$status_indicator,
|
||||
$log->listing_key,
|
||||
$log->media_key,
|
||||
$log->response_time_ms
|
||||
);
|
||||
|
||||
if ($log->status_code) {
|
||||
$line .= " HTTP:{$log->status_code}";
|
||||
}
|
||||
|
||||
if ($log->error_message) {
|
||||
$line .= " - {$log->error_message}";
|
||||
}
|
||||
|
||||
WP_CLI::line($line);
|
||||
}
|
||||
|
||||
WP_CLI::line('');
|
||||
WP_CLI::line(sprintf('Showing %d most recent entries. Use --limit=N to see more.', count($logs)));
|
||||
WP_CLI::line('');
|
||||
WP_CLI::success(sprintf('Cleared cached images for listing %s. They will be re-fetched on demand.', $listing_key));
|
||||
break;
|
||||
|
||||
default:
|
||||
WP_CLI::error("Unknown action: {$action}. Use 'status', 'process', 'reset', or 'logs'.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Output verbose media event information
|
||||
*
|
||||
* @param string $event Event name
|
||||
* @param array $data Event data
|
||||
*/
|
||||
private function output_verbose_media_event($event, $data) {
|
||||
$timestamp = date('H:i:s');
|
||||
|
||||
switch ($event) {
|
||||
case 'media_downloaded':
|
||||
$listing = $data['listing_key'] ?? 'unknown';
|
||||
$key = $data['media_key'] ?? 'unknown';
|
||||
WP_CLI::line("[{$timestamp}] DOWNLOADED: {$listing} / {$key}");
|
||||
break;
|
||||
|
||||
case 'media_backoff':
|
||||
$listing = $data['listing_key'] ?? 'unknown';
|
||||
$key = $data['media_key'] ?? 'unknown';
|
||||
WP_CLI::warning("[{$timestamp}] BACKOFF: {$listing} / {$key} - will retry in 3 hours");
|
||||
break;
|
||||
|
||||
case 'media_error':
|
||||
$listing = $data['listing_key'] ?? 'unknown';
|
||||
$key = $data['media_key'] ?? 'unknown';
|
||||
$error = $data['error'] ?? 'Unknown error';
|
||||
WP_CLI::error("[{$timestamp}] ERROR: {$listing} / {$key} - {$error}", false);
|
||||
break;
|
||||
WP_CLI::error("Unknown action: {$action}. Use 'status', 'fetch', or 'clear'.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,9 +17,8 @@ All tables use `{$wpdb->prefix}mls_` prefix:
|
||||
|
||||
| Table | Purpose |
|
||||
|-------|---------|
|
||||
| `mls_properties` | Listing data |
|
||||
| `mls_media` | Media files with download queue |
|
||||
| `mls_media_log` | Media download attempt history |
|
||||
| `mls_properties` | Listing data (Active/Pending only) |
|
||||
| `mls_media` | Media metadata and cache status |
|
||||
| `mls_sync_state` | Sync progress tracking |
|
||||
| `mls_rate_limits` | API usage tracking |
|
||||
| `mls_sync_log` | Debug logging |
|
||||
@@ -40,7 +39,7 @@ MUST comply with these limits:
|
||||
- 40,000 requests/day
|
||||
- 4GB data/hour
|
||||
|
||||
Media downloads use 700ms delay (25% buffer) between requests.
|
||||
**Important**: The API rejects `$skip` values over ~80,000. Always use `@odata.nextLink` for pagination, never manual `$skip`.
|
||||
|
||||
### Key Files
|
||||
|
||||
@@ -48,7 +47,7 @@ Media downloads use 700ms delay (25% buffer) between requests.
|
||||
|------|---------|
|
||||
| `includes/class-mls-api-client.php` | API communication, auth, gzip |
|
||||
| `includes/class-mls-sync-engine.php` | Sync orchestration |
|
||||
| `includes/class-mls-media-handler.php` | Media queue and download |
|
||||
| `includes/class-mls-media-handler.php` | On-demand media fetch and cache |
|
||||
| `includes/class-mls-query.php` | Public query API |
|
||||
| `includes/class-mls-rate-limiter.php` | Rate limit compliance |
|
||||
| `cli/class-mls-cli.php` | WP-CLI commands |
|
||||
@@ -64,18 +63,16 @@ wp mls test auth
|
||||
wp mls status
|
||||
wp mls status rate-limits
|
||||
|
||||
# Run property sync (queues media, does not download)
|
||||
wp mls sync full [--dry-run] [--limit=N] [--verbose]
|
||||
wp mls sync incremental [--dry-run] [--verbose]
|
||||
# Run property sync
|
||||
wp mls sync full [--dry-run] [--limit=N] [--verbose] # Initial: Active/Pending only
|
||||
wp mls sync incremental [--dry-run] [--verbose] # Replication: all changes
|
||||
wp mls sync resume --id=<sync_id>
|
||||
|
||||
# Media download queue (separate from property sync)
|
||||
wp mls media status # Show queue stats
|
||||
wp mls media process # Download queued media (rate limited)
|
||||
wp mls media process --limit=50 --verbose
|
||||
wp mls media reset # Reset failed downloads for retry
|
||||
wp mls media logs # View download history
|
||||
wp mls media logs --clear --days=7
|
||||
# Media cache (images fetched on-demand when viewed)
|
||||
wp mls media status # Show cache statistics
|
||||
wp mls media fetch --listing=<key> # Pre-cache images for a listing
|
||||
wp mls media fetch --listing=<key> --limit=10 # Fetch up to 10 images
|
||||
wp mls media clear --listing=<key> # Clear cached images for re-fetch
|
||||
|
||||
# Statistics
|
||||
wp mls stats
|
||||
@@ -83,9 +80,6 @@ wp mls stats
|
||||
# Cache management
|
||||
wp mls cache clear --confirm
|
||||
wp mls cache cleanup
|
||||
wp mls cache missing # View failed media downloads
|
||||
wp mls cache missing --limit=20 # View first 20 entries
|
||||
wp mls cache missing --clear # Clear the log
|
||||
|
||||
# Recovery commands
|
||||
wp mls recovery list # Show resumable syncs
|
||||
@@ -93,26 +87,64 @@ wp mls recovery auto # Auto-resume most recent failed sync
|
||||
wp mls recovery cleanup # Mark stale (>1hr) syncs as failed
|
||||
```
|
||||
|
||||
### Media Queue System
|
||||
## Sync Strategy (IMPORTANT)
|
||||
|
||||
Media downloads are now queue-based and separate from property sync:
|
||||
The sync follows MLS Grid best practices for replication:
|
||||
|
||||
1. **Property sync** (`wp mls sync full/incremental`) queues media records
|
||||
2. **Media process** (`wp mls media process`) downloads queued media with rate limiting
|
||||
3. Downloads are rate-limited to 700ms between requests (under 2/sec limit)
|
||||
4. Failed downloads get 3-hour backoff before retry
|
||||
5. After 5 attempts, items are marked failed and logged
|
||||
### Initial Import (`wp mls sync full`)
|
||||
|
||||
**Queue states:**
|
||||
- `pending` - Ready for download
|
||||
- `completed` - Successfully downloaded
|
||||
- `failed` - Max attempts reached
|
||||
- Fetches ONLY `Active` and `Pending` properties
|
||||
- Filter: `MlgCanView eq true and (StandardStatus eq 'Active' or StandardStatus eq 'Pending')`
|
||||
- Uses `@odata.nextLink` for pagination (NOT `$skip`)
|
||||
- Stores media metadata but does NOT download images
|
||||
- ~30,000 records for NorthStar MLS (vs 1.3M total including Closed)
|
||||
|
||||
**Media table columns:**
|
||||
- `download_status` - pending/completed/failed
|
||||
- `retry_after` - Next retry time (3hr backoff on rate limit)
|
||||
- `queued_at` - When item was queued
|
||||
- `download_attempts` - Attempt count (max 5)
|
||||
### Replication (`wp mls sync incremental`)
|
||||
|
||||
- Fetches ALL properties modified since last sync
|
||||
- NO filter on `MlgCanView` or `StandardStatus` - we need to see changes
|
||||
- For each record received:
|
||||
- If `MlgCanView = false` -> DELETE from local DB
|
||||
- If `StandardStatus` not in (Active, Pending) -> DELETE from local DB
|
||||
- Otherwise -> INSERT or UPDATE
|
||||
- This handles: new listings, price changes, status changes (Active->Sold), removals
|
||||
|
||||
### Why This Approach?
|
||||
|
||||
1. **MLS Grid API limits `$skip` to ~80,000** - bulk scanning all 1.3M records fails
|
||||
2. **We only care about available properties** - no need to store Closed/Sold
|
||||
3. **Replication is efficient** - only fetches changed records
|
||||
4. **Proper deletion handling** - when a property sells, we remove it
|
||||
|
||||
### Data Flow
|
||||
|
||||
```
|
||||
Initial Import:
|
||||
API (Active/Pending + MlgCanView=true) -> Local DB
|
||||
|
||||
Replication (every 15 min):
|
||||
API (ModificationTimestamp > last_sync) -> Check each record:
|
||||
- MlgCanView=false OR Status!=Active/Pending -> DELETE locally
|
||||
- Otherwise -> UPSERT locally
|
||||
```
|
||||
|
||||
## Media System (On-Demand Fetching)
|
||||
|
||||
Per MLS Grid rules, media URLs must NOT be used directly on websites. Images must be downloaded and served from our own server.
|
||||
|
||||
**How it works:**
|
||||
1. **Property sync** stores media metadata (URLs, keys, order) but does NOT download images
|
||||
2. **On-demand fetch**: When `mls_get_property_image()` is called, the image is fetched and cached locally
|
||||
3. **Subsequent requests** serve from local cache
|
||||
4. **Pre-caching**: Use `wp mls media fetch --listing=<key>` to pre-cache specific listings
|
||||
|
||||
**Benefits:**
|
||||
- No rate limit issues from bulk downloading
|
||||
- Images cached only when needed (saves bandwidth/storage)
|
||||
- Automatic re-fetch if cache is cleared
|
||||
- Works with MLS Grid's image URL expiration
|
||||
|
||||
**Cache location:** `wp-content/uploads/mls-listings/{prefix}/{listing_key}/`
|
||||
|
||||
### Progress Output
|
||||
|
||||
@@ -121,23 +153,10 @@ Property sync (compact mode):
|
||||
- `#` = property updated
|
||||
- `x` = property deleted
|
||||
- `-` = skipped (dry-run)
|
||||
- `q` = media queued
|
||||
- `p` = media skipped (already downloaded)
|
||||
- `|` = page complete
|
||||
|
||||
Media process (compact mode):
|
||||
- `P` = downloaded
|
||||
- `B` = backoff (retry later)
|
||||
- `E` = error
|
||||
|
||||
With --verbose: Full timestamped output.
|
||||
|
||||
### Missing Media Log
|
||||
|
||||
Permanently failed media downloads logged to: `wp-content/uploads/mls-missing-media.log`
|
||||
|
||||
Format: `[timestamp] listing_key | media_key | error | url`
|
||||
|
||||
### Sync Recovery
|
||||
|
||||
The sync engine saves progress after each page:
|
||||
@@ -152,16 +171,15 @@ The sync engine saves progress after each page:
|
||||
### Recommended Cron Setup
|
||||
|
||||
```bash
|
||||
# Property sync every 30 minutes
|
||||
*/30 * * * * cd /var/www/html && wp mls recovery auto --quiet && wp mls sync incremental --allow-root >> /var/log/mls-sync.log 2>&1
|
||||
# Replication sync every 15 minutes (MLS Grid recommended)
|
||||
*/15 * * * * cd /var/www/html && wp mls sync incremental --allow-root >> /var/log/mls-sync.log 2>&1
|
||||
|
||||
# Media downloads every 5 minutes (processes up to 50 items per run)
|
||||
*/5 * * * * cd /var/www/html && wp mls media process --limit=50 --quiet --allow-root >> /var/log/mls-media.log 2>&1
|
||||
|
||||
# Full sync weekly (Sunday 3am)
|
||||
0 3 * * 0 cd /var/www/html && wp mls sync full --allow-root >> /var/log/mls-sync.log 2>&1
|
||||
# Full re-sync weekly (Sunday 3am) - rebuilds from scratch
|
||||
0 3 * * 0 cd /var/www/html && wp mls cache clear --confirm --allow-root && wp mls sync full --allow-root >> /var/log/mls-sync.log 2>&1
|
||||
```
|
||||
|
||||
Note: No separate media cron needed - images are fetched on-demand when properties are viewed.
|
||||
|
||||
### Public API Functions
|
||||
|
||||
Available for themes/plugins:
|
||||
@@ -178,9 +196,19 @@ $properties = mls_get_properties([
|
||||
// Get single property
|
||||
$property = mls_get_property('NST123456');
|
||||
|
||||
// Get media
|
||||
// Get media (on-demand fetching)
|
||||
$image_url = mls_get_property_image('NST123456'); // Fetches if not cached
|
||||
$image_url = mls_get_property_image('NST123456', false); // Return null if not cached
|
||||
|
||||
// Get all images (fetches first N on demand)
|
||||
$images = mls_get_property_images('NST123456'); // Fetches first 1 if uncached
|
||||
$images = mls_get_property_images('NST123456', 5); // Fetches first 5 if uncached
|
||||
|
||||
// Get media metadata (no fetch)
|
||||
$media = mls_get_property_media('NST123456');
|
||||
$image_url = mls_get_property_image('NST123456');
|
||||
|
||||
// Get cache statistics
|
||||
$stats = mls_get_cache_stats(); // Returns total_media, cached, uncached counts
|
||||
|
||||
// Get distinct values
|
||||
$cities = mls_get_cities('Active');
|
||||
@@ -189,20 +217,12 @@ $cities = mls_get_cities('Active');
|
||||
if (mls_is_available()) { ... }
|
||||
```
|
||||
|
||||
### Sync Strategy
|
||||
|
||||
1. **Property Sync**: Full/incremental sync downloads property data and queues media
|
||||
2. **Media Queue**: Separate process downloads media with rate limiting
|
||||
3. **Delete Handling**: MlgCanView=false triggers local deletion
|
||||
4. **Media Storage**: Downloads to wp-content/uploads/mls-listings/
|
||||
5. **Recovery**: Stores last_next_link for resume on failure
|
||||
|
||||
### Testing After Changes
|
||||
|
||||
```bash
|
||||
wp mls test connection
|
||||
wp mls test auth
|
||||
wp mls sync full --dry-run --limit=10
|
||||
wp mls sync full --dry-run --limit=10 --verbose
|
||||
wp mls media status
|
||||
wp mls stats
|
||||
```
|
||||
@@ -226,3 +246,28 @@ Key fields from API to database:
|
||||
| MlgCanView | mlg_can_view |
|
||||
|
||||
Full API response stored in `raw_data` column as JSON.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "Value out of range" error
|
||||
The API is rejecting a high `$skip` value. This means pagination broke. Clear data and re-run initial sync:
|
||||
```bash
|
||||
wp mls cache clear --confirm --allow-root
|
||||
wp mls sync full --allow-root
|
||||
```
|
||||
|
||||
### All properties showing as "Sold"
|
||||
The initial sync was run without the Active/Pending filter. Clear and re-sync:
|
||||
```bash
|
||||
wp mls cache clear --confirm --allow-root
|
||||
wp mls sync full --allow-root
|
||||
```
|
||||
|
||||
### Media not loading
|
||||
Images are fetched on-demand. Check:
|
||||
1. `wp mls media status` - see cache stats
|
||||
2. `wp mls media fetch --listing=<key>` - manually fetch for a listing
|
||||
3. Check `wp-content/uploads/mls-listings/` directory permissions
|
||||
|
||||
### Sync taking too long
|
||||
Initial sync of ~30K Active/Pending properties takes about 30-45 minutes. Use `--verbose` to see progress.
|
||||
|
||||
@@ -318,26 +318,53 @@ class MLS_API_Client {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get properties including those marked for deletion (for sync)
|
||||
* Get properties for initial sync (Active/Pending only)
|
||||
*
|
||||
* @param string|null $timestamp Optional modification timestamp filter
|
||||
* @param string|null $expand Expand parameter
|
||||
* @param int|null $top Number of records
|
||||
* @return array|WP_Error Response data or error
|
||||
*/
|
||||
public function get_properties_for_sync($timestamp = null, $expand = null, $top = null) {
|
||||
// Don't filter by MlgCanView for sync - we need to see deleted records
|
||||
public function get_properties_for_initial_sync($expand = null, $top = null) {
|
||||
$params = array();
|
||||
|
||||
$system = $this->options->get_originating_system();
|
||||
|
||||
if ($timestamp) {
|
||||
$params['$filter'] = "OriginatingSystemName eq '{$system}' and ModificationTimestamp gt {$timestamp}";
|
||||
} else {
|
||||
// Initial sync - only get viewable records
|
||||
$params['$filter'] = "OriginatingSystemName eq '{$system}' and MlgCanView eq true";
|
||||
// Initial sync: only Active/Pending with MlgCanView=true
|
||||
$params['$filter'] = "OriginatingSystemName eq '{$system}' and MlgCanView eq true and (StandardStatus eq 'Active' or StandardStatus eq 'Pending')";
|
||||
|
||||
if ($expand) {
|
||||
$params['$expand'] = $expand;
|
||||
}
|
||||
|
||||
if ($top) {
|
||||
$params['$top'] = min($top, $expand ? self::MAX_TOP_WITH_EXPAND : self::MAX_TOP_NO_EXPAND);
|
||||
} else {
|
||||
$params['$top'] = $expand ? self::MAX_TOP_WITH_EXPAND : self::DEFAULT_TOP;
|
||||
}
|
||||
|
||||
return $this->request('Property', $params);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get properties modified since timestamp (for replication)
|
||||
*
|
||||
* Does NOT filter by MlgCanView or StandardStatus so we can detect:
|
||||
* - Records that became unavailable (MlgCanView=false)
|
||||
* - Records that changed status (Active -> Sold)
|
||||
*
|
||||
* @param string $timestamp ISO 8601 modification timestamp
|
||||
* @param string|null $expand Expand parameter
|
||||
* @param int|null $top Number of records
|
||||
* @return array|WP_Error Response data or error
|
||||
*/
|
||||
public function get_properties_for_replication($timestamp, $expand = null, $top = null) {
|
||||
$params = array();
|
||||
|
||||
$system = $this->options->get_originating_system();
|
||||
|
||||
// Replication: get ALL changes since timestamp (no MlgCanView or Status filter)
|
||||
$params['$filter'] = "OriginatingSystemName eq '{$system}' and ModificationTimestamp gt {$timestamp}";
|
||||
|
||||
if ($expand) {
|
||||
$params['$expand'] = $expand;
|
||||
}
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
/**
|
||||
* MLS Media Handler
|
||||
*
|
||||
* Handles downloading and managing media files from MLS listings
|
||||
* Uses a queue-based system with rate limiting to comply with API limits
|
||||
* Handles on-demand fetching and caching of media files from MLS listings.
|
||||
* Images are downloaded when first requested and cached locally.
|
||||
*
|
||||
* Rate limits: 2 requests/second (500ms minimum between requests)
|
||||
* We use 700ms between requests (25% buffer)
|
||||
* Per MLS Grid rules:
|
||||
* - MediaURLs must NOT be used directly on websites
|
||||
* - Images must be downloaded and served from our own server
|
||||
*/
|
||||
|
||||
if (!defined('ABSPATH')) {
|
||||
@@ -20,21 +21,6 @@ class MLS_Media_Handler {
|
||||
*/
|
||||
const UPLOAD_SUBDIR = 'mls-listings';
|
||||
|
||||
/**
|
||||
* Minimum delay between media downloads in milliseconds (700ms = 25% buffer over 500ms limit)
|
||||
*/
|
||||
const DOWNLOAD_DELAY_MS = 700;
|
||||
|
||||
/**
|
||||
* Retry backoff time in hours for failed downloads
|
||||
*/
|
||||
const RETRY_BACKOFF_HOURS = 3;
|
||||
|
||||
/**
|
||||
* Maximum download attempts before permanent failure
|
||||
*/
|
||||
const MAX_ATTEMPTS = 5;
|
||||
|
||||
/**
|
||||
* Database instance
|
||||
*/
|
||||
@@ -80,28 +66,28 @@ class MLS_Media_Handler {
|
||||
* @return string Absolute path
|
||||
*/
|
||||
public function get_listing_dir($listing_key) {
|
||||
// Use first 2 characters as subdirectory to prevent too many files in one folder
|
||||
$prefix = substr($listing_key, 0, 2);
|
||||
return $this->get_upload_dir() . '/' . $prefix . '/' . $listing_key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue media for a property (does NOT download immediately)
|
||||
* Store media metadata from API sync (no download)
|
||||
*
|
||||
* @param string $listing_key Listing key
|
||||
* @param array $media_array Media array from API
|
||||
* @param callable|null $progress_callback Callback for progress updates
|
||||
* @return array Stats
|
||||
*/
|
||||
public function queue_property_media($listing_key, $media_array, $progress_callback = null) {
|
||||
public function sync_property_media($listing_key, $media_array, $force = false, $progress_callback = null) {
|
||||
global $wpdb;
|
||||
|
||||
if (empty($media_array)) {
|
||||
return;
|
||||
return array('stored' => 0, 'skipped' => 0);
|
||||
}
|
||||
|
||||
$received_keys = array();
|
||||
$queued_count = 0;
|
||||
$skipped_count = 0;
|
||||
$stored = 0;
|
||||
$skipped = 0;
|
||||
|
||||
foreach ($media_array as $media) {
|
||||
$media_key = $media['MediaKey'] ?? null;
|
||||
@@ -134,40 +120,35 @@ class MLS_Media_Handler {
|
||||
);
|
||||
|
||||
if ($existing) {
|
||||
// Update existing record
|
||||
// Check if URL changed - if so, clear cached file
|
||||
if ($existing->media_url !== ($media['MediaURL'] ?? null) && $existing->local_path) {
|
||||
$file_path = $this->get_upload_dir() . '/' . $existing->local_path;
|
||||
if (file_exists($file_path)) {
|
||||
unlink($file_path);
|
||||
}
|
||||
$data['local_path'] = null;
|
||||
$data['local_url'] = null;
|
||||
$data['downloaded_at'] = null;
|
||||
}
|
||||
|
||||
$wpdb->update(
|
||||
$this->db->media_table(),
|
||||
$data,
|
||||
array('id' => $existing->id)
|
||||
);
|
||||
|
||||
// Check if we need to re-download (queue it)
|
||||
if ($this->needs_download($existing, $media)) {
|
||||
$this->mark_for_download($existing->id);
|
||||
$queued_count++;
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, 'media_queued', array('media_key' => $media_key));
|
||||
}
|
||||
$skipped++;
|
||||
} else {
|
||||
$skipped_count++;
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, 'media_skipped', array('media_key' => $media_key));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Insert new record - queued for download
|
||||
$data['created_at'] = current_time('mysql');
|
||||
$data['queued_at'] = current_time('mysql');
|
||||
$data['download_status'] = 'pending';
|
||||
$wpdb->insert($this->db->media_table(), $data);
|
||||
$queued_count++;
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, 'media_queued', array('media_key' => $media_key));
|
||||
$stored++;
|
||||
}
|
||||
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, 'media_stored', array('media_key' => $media_key));
|
||||
}
|
||||
}
|
||||
|
||||
// Delete media that no longer exists
|
||||
// Delete orphaned media records
|
||||
if (!empty($received_keys)) {
|
||||
$placeholders = implode(',', array_fill(0, count($received_keys), '%s'));
|
||||
$values = array_merge(array($listing_key), $received_keys);
|
||||
@@ -179,347 +160,203 @@ class MLS_Media_Handler {
|
||||
));
|
||||
|
||||
foreach ($orphaned as $record) {
|
||||
// Delete file if exists
|
||||
if ($record->local_path) {
|
||||
$file_path = $this->get_upload_dir() . '/' . $record->local_path;
|
||||
if (file_exists($file_path)) {
|
||||
unlink($file_path);
|
||||
}
|
||||
}
|
||||
|
||||
// Delete record
|
||||
$wpdb->delete($this->db->media_table(), array('id' => $record->id));
|
||||
}
|
||||
}
|
||||
|
||||
return array(
|
||||
'queued' => $queued_count,
|
||||
'skipped' => $skipped_count,
|
||||
);
|
||||
return array('stored' => $stored, 'skipped' => $skipped);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a media record for download
|
||||
* Get image URL for a media record, fetching on-demand if needed
|
||||
*
|
||||
* @param int $media_id Media ID
|
||||
* @param int|object $media Media ID or media record object
|
||||
* @param bool $fetch_if_missing Whether to fetch if not cached
|
||||
* @return string|null Local URL or null
|
||||
*/
|
||||
private function mark_for_download($media_id) {
|
||||
global $wpdb;
|
||||
|
||||
$wpdb->update(
|
||||
$this->db->media_table(),
|
||||
array(
|
||||
'download_status' => 'pending',
|
||||
'queued_at' => current_time('mysql'),
|
||||
'local_path' => null,
|
||||
'local_url' => null,
|
||||
'downloaded_at' => null,
|
||||
'download_error' => null,
|
||||
),
|
||||
array('id' => $media_id)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if media needs to be downloaded
|
||||
*
|
||||
* @param object $existing Existing media record
|
||||
* @param array $new_data New media data from API
|
||||
* @return bool
|
||||
*/
|
||||
private function needs_download($existing, $new_data) {
|
||||
// No local file
|
||||
if (empty($existing->local_path)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// File doesn't exist
|
||||
$file_path = $this->get_upload_dir() . '/' . $existing->local_path;
|
||||
if (!file_exists($file_path)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Media URL changed
|
||||
if ($existing->media_url !== ($new_data['MediaURL'] ?? null)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the next media item to download from the queue
|
||||
*
|
||||
* @return object|null Media record or null if queue is empty
|
||||
*/
|
||||
public function get_next_queued() {
|
||||
global $wpdb;
|
||||
|
||||
$now = current_time('mysql');
|
||||
|
||||
// Get next pending item that's not in retry backoff
|
||||
return $wpdb->get_row($wpdb->prepare(
|
||||
"SELECT * FROM {$this->db->media_table()}
|
||||
WHERE download_status = 'pending'
|
||||
AND media_url IS NOT NULL
|
||||
AND download_attempts < %d
|
||||
AND (retry_after IS NULL OR retry_after <= %s)
|
||||
ORDER BY queued_at ASC
|
||||
LIMIT 1",
|
||||
self::MAX_ATTEMPTS,
|
||||
$now
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue statistics
|
||||
*
|
||||
* @return array Queue stats
|
||||
*/
|
||||
public function get_queue_stats() {
|
||||
global $wpdb;
|
||||
|
||||
$now = current_time('mysql');
|
||||
|
||||
return array(
|
||||
'pending' => (int) $wpdb->get_var(
|
||||
"SELECT COUNT(*) FROM {$this->db->media_table()}
|
||||
WHERE download_status = 'pending'"
|
||||
),
|
||||
'ready' => (int) $wpdb->get_var($wpdb->prepare(
|
||||
"SELECT COUNT(*) FROM {$this->db->media_table()}
|
||||
WHERE download_status = 'pending'
|
||||
AND media_url IS NOT NULL
|
||||
AND download_attempts < %d
|
||||
AND (retry_after IS NULL OR retry_after <= %s)",
|
||||
self::MAX_ATTEMPTS,
|
||||
$now
|
||||
)),
|
||||
'in_backoff' => (int) $wpdb->get_var($wpdb->prepare(
|
||||
"SELECT COUNT(*) FROM {$this->db->media_table()}
|
||||
WHERE download_status = 'pending'
|
||||
AND retry_after > %s",
|
||||
$now
|
||||
)),
|
||||
'failed' => (int) $wpdb->get_var($wpdb->prepare(
|
||||
"SELECT COUNT(*) FROM {$this->db->media_table()}
|
||||
WHERE download_status = 'failed'
|
||||
OR download_attempts >= %d",
|
||||
self::MAX_ATTEMPTS
|
||||
)),
|
||||
'completed' => (int) $wpdb->get_var(
|
||||
"SELECT COUNT(*) FROM {$this->db->media_table()}
|
||||
WHERE download_status = 'completed'"
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process media queue with rate limiting
|
||||
*
|
||||
* @param int $limit Max items to process
|
||||
* @param callable|null $progress_callback Callback for progress updates
|
||||
* @return array Processing stats
|
||||
*/
|
||||
public function process_queue($limit = 100, $progress_callback = null) {
|
||||
$stats = array(
|
||||
'processed' => 0,
|
||||
'success' => 0,
|
||||
'failed' => 0,
|
||||
'skipped' => 0,
|
||||
);
|
||||
|
||||
$last_download_time = 0;
|
||||
|
||||
for ($i = 0; $i < $limit; $i++) {
|
||||
$media = $this->get_next_queued();
|
||||
|
||||
if (!$media) {
|
||||
// Queue empty
|
||||
break;
|
||||
}
|
||||
|
||||
// Rate limiting: ensure minimum delay between downloads
|
||||
$now_ms = microtime(true) * 1000;
|
||||
$elapsed = $now_ms - $last_download_time;
|
||||
|
||||
if ($elapsed < self::DOWNLOAD_DELAY_MS && $last_download_time > 0) {
|
||||
$wait_ms = (int) (self::DOWNLOAD_DELAY_MS - $elapsed);
|
||||
usleep($wait_ms * 1000);
|
||||
}
|
||||
|
||||
// Download the media
|
||||
$result = $this->download_media($media->id);
|
||||
$last_download_time = microtime(true) * 1000;
|
||||
|
||||
$stats['processed']++;
|
||||
|
||||
if ($result === true) {
|
||||
$stats['success']++;
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, 'media_downloaded', array(
|
||||
'media_key' => $media->media_key,
|
||||
'listing_key' => $media->listing_key,
|
||||
));
|
||||
}
|
||||
} elseif ($result === 'backoff') {
|
||||
$stats['skipped']++;
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, 'media_backoff', array(
|
||||
'media_key' => $media->media_key,
|
||||
'listing_key' => $media->listing_key,
|
||||
));
|
||||
}
|
||||
} else {
|
||||
$stats['failed']++;
|
||||
if ($progress_callback) {
|
||||
call_user_func($progress_callback, 'media_error', array(
|
||||
'media_key' => $media->media_key,
|
||||
'listing_key' => $media->listing_key,
|
||||
'error' => $result,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a media file
|
||||
*
|
||||
* @param int $media_id Media record ID
|
||||
* @return bool|string True on success, 'backoff' if set for retry, error message on failure
|
||||
*/
|
||||
public function download_media($media_id) {
|
||||
public function get_image_url($media, $fetch_if_missing = true) {
|
||||
global $wpdb;
|
||||
|
||||
// Get media record if ID passed
|
||||
if (is_numeric($media)) {
|
||||
$media = $wpdb->get_row($wpdb->prepare(
|
||||
"SELECT * FROM {$this->db->media_table()} WHERE id = %d",
|
||||
$media_id
|
||||
$media
|
||||
));
|
||||
|
||||
if (!$media || empty($media->media_url)) {
|
||||
return 'No media URL';
|
||||
}
|
||||
|
||||
// Increment attempt counter
|
||||
$wpdb->update(
|
||||
$this->db->media_table(),
|
||||
array('download_attempts' => $media->download_attempts + 1),
|
||||
array('id' => $media_id)
|
||||
);
|
||||
if (!$media) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Make the request
|
||||
$start_time = microtime(true);
|
||||
// Already cached
|
||||
if ($media->local_url && $media->local_path) {
|
||||
$file_path = $this->get_upload_dir() . '/' . $media->local_path;
|
||||
if (file_exists($file_path)) {
|
||||
return $media->local_url;
|
||||
}
|
||||
}
|
||||
|
||||
$response = wp_remote_get($media->media_url, array(
|
||||
'timeout' => 60,
|
||||
'stream' => false,
|
||||
// Fetch on demand
|
||||
if ($fetch_if_missing && $media->media_url) {
|
||||
$result = $this->fetch_and_cache($media);
|
||||
if ($result) {
|
||||
return $result;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get primary image URL for a listing (on-demand)
|
||||
*
|
||||
* @param string $listing_key Listing key
|
||||
* @param bool $fetch_if_missing Whether to fetch if not cached
|
||||
* @return string|null Image URL
|
||||
*/
|
||||
public function get_primary_image($listing_key, $fetch_if_missing = true) {
|
||||
global $wpdb;
|
||||
|
||||
// First check for already-cached image
|
||||
$cached = $wpdb->get_row($wpdb->prepare(
|
||||
"SELECT * FROM {$this->db->media_table()}
|
||||
WHERE listing_key = %s AND local_url IS NOT NULL AND local_path IS NOT NULL
|
||||
ORDER BY media_order ASC
|
||||
LIMIT 1",
|
||||
$listing_key
|
||||
));
|
||||
|
||||
$response_time_ms = (int) ((microtime(true) - $start_time) * 1000);
|
||||
$status_code = 0;
|
||||
$error_msg = null;
|
||||
if ($cached) {
|
||||
$file_path = $this->get_upload_dir() . '/' . $cached->local_path;
|
||||
if (file_exists($file_path)) {
|
||||
return $cached->local_url;
|
||||
}
|
||||
}
|
||||
|
||||
// Get first media record (may not be cached)
|
||||
$media = $wpdb->get_row($wpdb->prepare(
|
||||
"SELECT * FROM {$this->db->media_table()}
|
||||
WHERE listing_key = %s AND media_url IS NOT NULL
|
||||
ORDER BY media_order ASC
|
||||
LIMIT 1",
|
||||
$listing_key
|
||||
));
|
||||
|
||||
if (!$media) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// If already cached and file exists, return it
|
||||
if ($media->local_url && $media->local_path) {
|
||||
$file_path = $this->get_upload_dir() . '/' . $media->local_path;
|
||||
if (file_exists($file_path)) {
|
||||
return $media->local_url;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch on demand
|
||||
if ($fetch_if_missing) {
|
||||
return $this->fetch_and_cache($media);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all images for a listing (on-demand for first N)
|
||||
*
|
||||
* @param string $listing_key Listing key
|
||||
* @param int $fetch_limit Max images to fetch on-demand (0 = none)
|
||||
* @return array Media records with local_url populated where available
|
||||
*/
|
||||
public function get_listing_images($listing_key, $fetch_limit = 1) {
|
||||
global $wpdb;
|
||||
|
||||
$media = $wpdb->get_results($wpdb->prepare(
|
||||
"SELECT * FROM {$this->db->media_table()}
|
||||
WHERE listing_key = %s
|
||||
ORDER BY media_order ASC",
|
||||
$listing_key
|
||||
));
|
||||
|
||||
if (empty($media)) {
|
||||
return array();
|
||||
}
|
||||
|
||||
$fetched = 0;
|
||||
foreach ($media as &$item) {
|
||||
// Check if cached and file exists
|
||||
if ($item->local_url && $item->local_path) {
|
||||
$file_path = $this->get_upload_dir() . '/' . $item->local_path;
|
||||
if (file_exists($file_path)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch on demand up to limit
|
||||
if ($fetched < $fetch_limit && $item->media_url) {
|
||||
$url = $this->fetch_and_cache($item);
|
||||
if ($url) {
|
||||
$item->local_url = $url;
|
||||
$fetched++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $media;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch image from MLS Grid and cache locally
|
||||
*
|
||||
* @param object $media Media record
|
||||
* @return string|null Local URL on success, null on failure
|
||||
*/
|
||||
private function fetch_and_cache($media) {
|
||||
global $wpdb;
|
||||
|
||||
if (empty($media->media_url)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Download the image
|
||||
$response = wp_remote_get($media->media_url, array(
|
||||
'timeout' => 30,
|
||||
));
|
||||
|
||||
if (is_wp_error($response)) {
|
||||
$error_msg = $response->get_error_message();
|
||||
$this->log_download($media, 'error', null, $response_time_ms, $error_msg);
|
||||
$this->handle_download_failure($media_id, $error_msg, false);
|
||||
return $error_msg;
|
||||
$this->logger->warning('Media fetch failed', array(
|
||||
'listing_key' => $media->listing_key,
|
||||
'media_key' => $media->media_key,
|
||||
'error' => $response->get_error_message(),
|
||||
));
|
||||
return null;
|
||||
}
|
||||
|
||||
$status_code = wp_remote_retrieve_response_code($response);
|
||||
$this->log_download($media, 'attempt', $status_code, $response_time_ms, null);
|
||||
|
||||
// Success
|
||||
if ($status_code === 200) {
|
||||
if ($status_code !== 200) {
|
||||
$this->logger->warning('Media fetch HTTP error', array(
|
||||
'listing_key' => $media->listing_key,
|
||||
'media_key' => $media->media_key,
|
||||
'status' => $status_code,
|
||||
));
|
||||
return null;
|
||||
}
|
||||
|
||||
$body = wp_remote_retrieve_body($response);
|
||||
if (empty($body)) {
|
||||
$error_msg = 'Empty response body';
|
||||
$this->log_download($media, 'error', $status_code, $response_time_ms, $error_msg);
|
||||
$this->handle_download_failure($media_id, $error_msg, false);
|
||||
return $error_msg;
|
||||
return null;
|
||||
}
|
||||
|
||||
// Save the file
|
||||
$save_result = $this->save_media_file($media, $body, $response);
|
||||
if ($save_result !== true) {
|
||||
$this->log_download($media, 'error', $status_code, $response_time_ms, $save_result);
|
||||
$this->handle_download_failure($media_id, $save_result, false);
|
||||
return $save_result;
|
||||
}
|
||||
|
||||
$this->log_download($media, 'success', $status_code, $response_time_ms, null);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Rate limited (429) or server error (5xx) - set backoff
|
||||
$retryable = in_array($status_code, array(429, 500, 502, 503, 504));
|
||||
$error_msg = "HTTP {$status_code}";
|
||||
|
||||
if ($retryable) {
|
||||
$this->log_download($media, 'rate_limited', $status_code, $response_time_ms, $error_msg);
|
||||
$this->handle_download_failure($media_id, $error_msg, true);
|
||||
return 'backoff';
|
||||
}
|
||||
|
||||
// Permanent failure (404, 403, etc.)
|
||||
$this->log_download($media, 'permanent_error', $status_code, $response_time_ms, $error_msg);
|
||||
$this->handle_download_failure($media_id, $error_msg, false);
|
||||
return $error_msg;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle download failure
|
||||
*
|
||||
* @param int $media_id Media ID
|
||||
* @param string $error Error message
|
||||
* @param bool $set_backoff Whether to set retry backoff
|
||||
*/
|
||||
private function handle_download_failure($media_id, $error, $set_backoff) {
|
||||
global $wpdb;
|
||||
|
||||
$media = $wpdb->get_row($wpdb->prepare(
|
||||
"SELECT * FROM {$this->db->media_table()} WHERE id = %d",
|
||||
$media_id
|
||||
));
|
||||
|
||||
$update_data = array(
|
||||
'download_error' => $error,
|
||||
);
|
||||
|
||||
if ($set_backoff) {
|
||||
// Set retry_after to 3 hours from now
|
||||
$retry_after = date('Y-m-d H:i:s', strtotime('+' . self::RETRY_BACKOFF_HOURS . ' hours'));
|
||||
$update_data['retry_after'] = $retry_after;
|
||||
}
|
||||
|
||||
// Check if max attempts reached
|
||||
if ($media && $media->download_attempts >= self::MAX_ATTEMPTS) {
|
||||
$update_data['download_status'] = 'failed';
|
||||
$this->log_missing_media($media, $error);
|
||||
}
|
||||
|
||||
$wpdb->update(
|
||||
$this->db->media_table(),
|
||||
$update_data,
|
||||
array('id' => $media_id)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save downloaded media file to disk
|
||||
*
|
||||
* @param object $media Media record
|
||||
* @param string $body File contents
|
||||
* @param array $response HTTP response
|
||||
* @return bool|string True on success, error message on failure
|
||||
*/
|
||||
private function save_media_file($media, $body, $response) {
|
||||
global $wpdb;
|
||||
|
||||
// Determine file extension from content type or URL
|
||||
// Determine extension
|
||||
$content_type = wp_remote_retrieve_header($response, 'content-type');
|
||||
$extension = $this->get_extension_from_content_type($content_type, $media->media_url);
|
||||
|
||||
@@ -534,15 +371,17 @@ class MLS_Media_Handler {
|
||||
$file_path = $listing_dir . '/' . $filename;
|
||||
|
||||
if (file_put_contents($file_path, $body) === false) {
|
||||
return 'Failed to write file';
|
||||
$this->logger->error('Failed to write media file', array(
|
||||
'path' => $file_path,
|
||||
));
|
||||
return null;
|
||||
}
|
||||
|
||||
// Calculate relative path
|
||||
// Update database
|
||||
$prefix = substr($media->listing_key, 0, 2);
|
||||
$relative_path = $prefix . '/' . $media->listing_key . '/' . $filename;
|
||||
$local_url = $this->get_upload_url() . '/' . $relative_path;
|
||||
|
||||
// Update record
|
||||
$wpdb->update(
|
||||
$this->db->media_table(),
|
||||
array(
|
||||
@@ -551,42 +390,17 @@ class MLS_Media_Handler {
|
||||
'file_size' => strlen($body),
|
||||
'mime_type' => $content_type,
|
||||
'downloaded_at' => current_time('mysql'),
|
||||
'download_error' => null,
|
||||
'download_status' => 'completed',
|
||||
'retry_after' => null,
|
||||
),
|
||||
array('id' => $media->id)
|
||||
);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a download attempt to the media log table
|
||||
*
|
||||
* @param object $media Media record
|
||||
* @param string $action Action type (attempt, success, error, rate_limited, permanent_error)
|
||||
* @param int|null $status_code HTTP status code
|
||||
* @param int $response_time_ms Response time in milliseconds
|
||||
* @param string|null $error Error message
|
||||
*/
|
||||
private function log_download($media, $action, $status_code, $response_time_ms, $error) {
|
||||
global $wpdb;
|
||||
|
||||
$wpdb->insert(
|
||||
$this->db->media_log_table(),
|
||||
array(
|
||||
'media_id' => $media->id,
|
||||
$this->logger->debug('Media fetched and cached', array(
|
||||
'listing_key' => $media->listing_key,
|
||||
'media_key' => $media->media_key,
|
||||
'action' => $action,
|
||||
'status_code' => $status_code,
|
||||
'response_time_ms' => $response_time_ms,
|
||||
'error_message' => $error,
|
||||
'url' => $media->media_url,
|
||||
'created_at' => current_time('mysql'),
|
||||
)
|
||||
);
|
||||
'size' => strlen($body),
|
||||
));
|
||||
|
||||
return $local_url;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -597,7 +411,6 @@ class MLS_Media_Handler {
|
||||
* @return string File extension
|
||||
*/
|
||||
private function get_extension_from_content_type($content_type, $url) {
|
||||
// Extract main type from content-type header
|
||||
$content_type = strtolower(explode(';', $content_type)[0]);
|
||||
|
||||
$map = array(
|
||||
@@ -619,60 +432,6 @@ class MLS_Media_Handler {
|
||||
return $ext ?: 'jpg';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path to the missing media log file
|
||||
*
|
||||
* @return string File path
|
||||
*/
|
||||
public function get_missing_log_path() {
|
||||
$upload_dir = wp_upload_dir();
|
||||
return $upload_dir['basedir'] . '/mls-missing-media.log';
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a failed media download to the missing media log file
|
||||
*
|
||||
* @param object $media Media record
|
||||
* @param string $error Error message
|
||||
*/
|
||||
private function log_missing_media($media, $error) {
|
||||
$log_file = $this->get_missing_log_path();
|
||||
$timestamp = date('Y-m-d H:i:s');
|
||||
$line = sprintf(
|
||||
"[%s] %s | %s | %s | %s\n",
|
||||
$timestamp,
|
||||
$media->listing_key,
|
||||
$media->media_key,
|
||||
$error,
|
||||
$media->media_url
|
||||
);
|
||||
file_put_contents($log_file, $line, FILE_APPEND | LOCK_EX);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the missing media log file
|
||||
*/
|
||||
public function clear_missing_log() {
|
||||
$log_file = $this->get_missing_log_path();
|
||||
if (file_exists($log_file)) {
|
||||
unlink($log_file);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get missing media count from log file
|
||||
*
|
||||
* @return int Number of missing media entries
|
||||
*/
|
||||
public function get_missing_count() {
|
||||
$log_file = $this->get_missing_log_path();
|
||||
if (!file_exists($log_file)) {
|
||||
return 0;
|
||||
}
|
||||
$content = file_get_contents($log_file);
|
||||
return substr_count($content, "\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all media for a property
|
||||
*
|
||||
@@ -719,7 +478,7 @@ class MLS_Media_Handler {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get media for a listing
|
||||
* Get media for a listing (legacy compatibility)
|
||||
*
|
||||
* @param string $listing_key Listing key
|
||||
* @return array Media records
|
||||
@@ -736,80 +495,13 @@ class MLS_Media_Handler {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get primary image URL for a listing
|
||||
* Clean up orphaned media files (files without database records)
|
||||
*
|
||||
* @param string $listing_key Listing key
|
||||
* @return string|null Image URL
|
||||
*/
|
||||
public function get_primary_image($listing_key) {
|
||||
global $wpdb;
|
||||
|
||||
$media = $wpdb->get_row($wpdb->prepare(
|
||||
"SELECT local_url, media_url FROM {$this->db->media_table()}
|
||||
WHERE listing_key = %s AND local_path IS NOT NULL
|
||||
ORDER BY media_order ASC
|
||||
LIMIT 1",
|
||||
$listing_key
|
||||
));
|
||||
|
||||
if ($media && $media->local_url) {
|
||||
return $media->local_url;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset failed downloads for retry
|
||||
*
|
||||
* @param string|null $listing_key Optional listing key to filter
|
||||
* @return int Number of records reset
|
||||
*/
|
||||
public function reset_failed_downloads($listing_key = null) {
|
||||
global $wpdb;
|
||||
|
||||
$where = "download_status = 'failed' OR download_attempts >= " . self::MAX_ATTEMPTS;
|
||||
$values = array();
|
||||
|
||||
if ($listing_key) {
|
||||
$where .= " AND listing_key = %s";
|
||||
$values[] = $listing_key;
|
||||
}
|
||||
|
||||
if (!empty($values)) {
|
||||
$sql = $wpdb->prepare(
|
||||
"UPDATE {$this->db->media_table()}
|
||||
SET download_status = 'pending',
|
||||
download_attempts = 0,
|
||||
download_error = NULL,
|
||||
retry_after = NULL,
|
||||
queued_at = %s
|
||||
WHERE {$where}",
|
||||
array_merge(array(current_time('mysql')), $values)
|
||||
);
|
||||
} else {
|
||||
$sql = $wpdb->prepare(
|
||||
"UPDATE {$this->db->media_table()}
|
||||
SET download_status = 'pending',
|
||||
download_attempts = 0,
|
||||
download_error = NULL,
|
||||
retry_after = NULL,
|
||||
queued_at = %s
|
||||
WHERE {$where}",
|
||||
current_time('mysql')
|
||||
);
|
||||
}
|
||||
|
||||
$wpdb->query($sql);
|
||||
return $wpdb->rows_affected;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up orphaned media (files without database records)
|
||||
*
|
||||
* @return int Number of files deleted
|
||||
* @return int Number of directories deleted
|
||||
*/
|
||||
public function cleanup_orphaned_files() {
|
||||
global $wpdb;
|
||||
|
||||
$deleted = 0;
|
||||
$base_dir = $this->get_upload_dir();
|
||||
|
||||
@@ -817,7 +509,6 @@ class MLS_Media_Handler {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Iterate through prefix directories
|
||||
foreach (scandir($base_dir) as $prefix) {
|
||||
if ($prefix === '.' || $prefix === '..' || !is_dir($base_dir . '/' . $prefix)) {
|
||||
continue;
|
||||
@@ -825,7 +516,6 @@ class MLS_Media_Handler {
|
||||
|
||||
$prefix_dir = $base_dir . '/' . $prefix;
|
||||
|
||||
// Iterate through listing directories
|
||||
foreach (scandir($prefix_dir) as $listing_key) {
|
||||
if ($listing_key === '.' || $listing_key === '..') {
|
||||
continue;
|
||||
@@ -836,8 +526,6 @@ class MLS_Media_Handler {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if listing exists in database
|
||||
global $wpdb;
|
||||
$exists = $wpdb->get_var($wpdb->prepare(
|
||||
"SELECT COUNT(*) FROM {$this->db->properties_table()} WHERE listing_key = %s",
|
||||
$listing_key
|
||||
@@ -854,76 +542,57 @@ class MLS_Media_Handler {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recent download logs
|
||||
* Get cache statistics
|
||||
*
|
||||
* @param int $limit Number of entries to return
|
||||
* @param string|null $action Optional action filter
|
||||
* @return array Log entries
|
||||
* @return array Cache stats
|
||||
*/
|
||||
public function get_download_logs($limit = 100, $action = null) {
|
||||
public function get_cache_stats() {
|
||||
global $wpdb;
|
||||
|
||||
$where = '';
|
||||
$values = array();
|
||||
|
||||
if ($action) {
|
||||
$where = "WHERE action = %s";
|
||||
$values[] = $action;
|
||||
}
|
||||
|
||||
$values[] = $limit;
|
||||
|
||||
return $wpdb->get_results($wpdb->prepare(
|
||||
"SELECT * FROM {$this->db->media_log_table()}
|
||||
{$where}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT %d",
|
||||
$values
|
||||
));
|
||||
return array(
|
||||
'total_media' => (int) $wpdb->get_var(
|
||||
"SELECT COUNT(*) FROM {$this->db->media_table()}"
|
||||
),
|
||||
'cached' => (int) $wpdb->get_var(
|
||||
"SELECT COUNT(*) FROM {$this->db->media_table()} WHERE local_url IS NOT NULL"
|
||||
),
|
||||
'uncached' => (int) $wpdb->get_var(
|
||||
"SELECT COUNT(*) FROM {$this->db->media_table()} WHERE local_url IS NULL"
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear old download logs
|
||||
* Get path to missing media log file (legacy compatibility)
|
||||
*
|
||||
* @param int $days_old Delete logs older than this many days
|
||||
* @return int Number of entries deleted
|
||||
* @return string File path
|
||||
*/
|
||||
public function clear_old_logs($days_old = 7) {
|
||||
global $wpdb;
|
||||
|
||||
$cutoff = date('Y-m-d H:i:s', strtotime("-{$days_old} days"));
|
||||
|
||||
$wpdb->query($wpdb->prepare(
|
||||
"DELETE FROM {$this->db->media_log_table()} WHERE created_at < %s",
|
||||
$cutoff
|
||||
));
|
||||
|
||||
return $wpdb->rows_affected;
|
||||
public function get_missing_log_path() {
|
||||
$upload_dir = wp_upload_dir();
|
||||
return $upload_dir['basedir'] . '/mls-missing-media.log';
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy sync method - now queues media instead of downloading immediately
|
||||
* Kept for backward compatibility
|
||||
* Get missing media count (legacy compatibility)
|
||||
*
|
||||
* @param string $listing_key Listing key
|
||||
* @param array $media_array Media array from API
|
||||
* @param bool $force Force re-download all media
|
||||
* @param callable|null $progress_callback Callback for progress updates
|
||||
* @return int
|
||||
*/
|
||||
public function sync_property_media($listing_key, $media_array, $force = false, $progress_callback = null) {
|
||||
// Now just queues media - actual download happens via process_queue()
|
||||
return $this->queue_property_media($listing_key, $media_array, $progress_callback);
|
||||
public function get_missing_count() {
|
||||
$log_file = $this->get_missing_log_path();
|
||||
if (!file_exists($log_file)) {
|
||||
return 0;
|
||||
}
|
||||
$content = file_get_contents($log_file);
|
||||
return substr_count($content, "\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Legacy download_pending method - now uses process_queue
|
||||
* Kept for backward compatibility
|
||||
*
|
||||
* @param int $limit Max media to download
|
||||
* @param callable|null $progress_callback Callback for progress updates
|
||||
* @return array Stats
|
||||
* Clear missing log (legacy compatibility)
|
||||
*/
|
||||
public function download_pending($limit = 100, $progress_callback = null) {
|
||||
return $this->process_queue($limit, $progress_callback);
|
||||
public function clear_missing_log() {
|
||||
$log_file = $this->get_missing_log_path();
|
||||
if (file_exists($log_file)) {
|
||||
unlink($log_file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +78,10 @@ class MLS_Sync_Engine {
|
||||
}
|
||||
|
||||
/**
|
||||
* Run full sync
|
||||
* Run full sync (Active/Pending properties only)
|
||||
*
|
||||
* Initial import fetches only Active and Pending properties.
|
||||
* Use incremental sync (replication) for ongoing updates.
|
||||
*
|
||||
* @param bool $dry_run If true, don't make changes
|
||||
* @param int|null $limit Max records to process
|
||||
@@ -86,7 +89,7 @@ class MLS_Sync_Engine {
|
||||
* @return array Sync results
|
||||
*/
|
||||
public function run_full_sync($dry_run = false, $limit = null, $progress_callback = null) {
|
||||
$this->logger->info('Starting full sync', array('dry_run' => $dry_run, 'limit' => $limit));
|
||||
$this->logger->info('Starting full sync (Active/Pending only)', array('dry_run' => $dry_run, 'limit' => $limit));
|
||||
|
||||
// Store progress callback for use in process_property
|
||||
$this->progress_callback = $progress_callback;
|
||||
@@ -106,15 +109,15 @@ class MLS_Sync_Engine {
|
||||
);
|
||||
|
||||
try {
|
||||
// Get first page of properties with media
|
||||
// Get first page of Active/Pending properties with media
|
||||
$start_time = microtime(true);
|
||||
$this->emit_progress('api_request', array(
|
||||
'method' => 'GET',
|
||||
'url' => 'Property',
|
||||
'params' => array('type' => 'full_sync', 'limit' => $limit),
|
||||
'params' => array('type' => 'initial_sync', 'filter' => 'Active/Pending', 'limit' => $limit),
|
||||
));
|
||||
|
||||
$response = $this->api_client->get_properties_for_sync(null, 'Media', $limit ? min($limit, 1000) : null);
|
||||
$response = $this->api_client->get_properties_for_initial_sync('Media', $limit ? min($limit, 1000) : null);
|
||||
$elapsed = round((microtime(true) - $start_time) * 1000);
|
||||
|
||||
if (is_wp_error($response)) {
|
||||
@@ -239,7 +242,13 @@ class MLS_Sync_Engine {
|
||||
}
|
||||
|
||||
/**
|
||||
* Run incremental sync
|
||||
* Run incremental sync (replication)
|
||||
*
|
||||
* Fetches all properties modified since last sync, including those that:
|
||||
* - Became unavailable (MlgCanView=false)
|
||||
* - Changed status (Active -> Sold)
|
||||
*
|
||||
* Properties are deleted from local DB if MlgCanView=false or status not Active/Pending.
|
||||
*
|
||||
* @param bool $dry_run If true, don't make changes
|
||||
* @param callable|null $progress_callback Callback for progress updates
|
||||
@@ -254,7 +263,7 @@ class MLS_Sync_Engine {
|
||||
return $this->run_full_sync($dry_run, null, $progress_callback);
|
||||
}
|
||||
|
||||
$this->logger->info('Starting incremental sync', array(
|
||||
$this->logger->info('Starting replication sync', array(
|
||||
'since' => $last_timestamp,
|
||||
'dry_run' => $dry_run,
|
||||
));
|
||||
@@ -276,15 +285,15 @@ class MLS_Sync_Engine {
|
||||
);
|
||||
|
||||
try {
|
||||
// Get modified properties (including those marked for deletion)
|
||||
// Get ALL modified properties (no MlgCanView or status filter for replication)
|
||||
$start_time = microtime(true);
|
||||
$this->emit_progress('api_request', array(
|
||||
'method' => 'GET',
|
||||
'url' => 'Property',
|
||||
'params' => array('type' => 'incremental', 'since' => $last_timestamp),
|
||||
'params' => array('type' => 'replication', 'since' => $last_timestamp),
|
||||
));
|
||||
|
||||
$response = $this->api_client->get_properties_since($last_timestamp, 'Media');
|
||||
$response = $this->api_client->get_properties_for_replication($last_timestamp, 'Media');
|
||||
$elapsed = round((microtime(true) - $start_time) * 1000);
|
||||
|
||||
if (is_wp_error($response)) {
|
||||
@@ -525,9 +534,18 @@ class MLS_Sync_Engine {
|
||||
*/
|
||||
private $progress_callback = null;
|
||||
|
||||
/**
|
||||
* Allowed statuses for our database (Active/Pending only)
|
||||
*/
|
||||
const ALLOWED_STATUSES = array('Active', 'Pending');
|
||||
|
||||
/**
|
||||
* Process a single property record
|
||||
*
|
||||
* During replication, properties are deleted if:
|
||||
* - MlgCanView = false (removed from feed)
|
||||
* - StandardStatus not in (Active, Pending)
|
||||
*
|
||||
* @param array $property Property data from API
|
||||
* @param bool $dry_run If true, don't make changes
|
||||
*/
|
||||
@@ -543,15 +561,31 @@ class MLS_Sync_Engine {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check MlgCanView - if false, delete the record
|
||||
// Check MlgCanView and StandardStatus
|
||||
$can_view = $property['MlgCanView'] ?? true;
|
||||
$status = $property['StandardStatus'] ?? null;
|
||||
|
||||
if (!$can_view) {
|
||||
// Delete if: not viewable OR status is not Active/Pending
|
||||
$should_delete = !$can_view || !in_array($status, self::ALLOWED_STATUSES);
|
||||
|
||||
if ($should_delete) {
|
||||
// Check if we have this record locally before attempting delete
|
||||
$exists_locally = $wpdb->get_var($wpdb->prepare(
|
||||
"SELECT id FROM {$this->db->properties_table()} WHERE listing_key = %s",
|
||||
$listing_key
|
||||
));
|
||||
|
||||
if ($exists_locally) {
|
||||
if (!$dry_run) {
|
||||
$this->delete_property($listing_key);
|
||||
}
|
||||
$this->stats['deleted']++;
|
||||
$this->emit_progress('property_deleted', array('listing_key' => $listing_key));
|
||||
$this->emit_progress('property_deleted', array(
|
||||
'listing_key' => $listing_key,
|
||||
'reason' => !$can_view ? 'MlgCanView=false' : "Status={$status}",
|
||||
));
|
||||
}
|
||||
// If not in our DB, just skip silently (e.g., Sold property we never had)
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -271,17 +271,21 @@ function mls_get_property_media($listing_key) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get primary image URL for a listing
|
||||
* Get primary image URL for a listing (on-demand fetching)
|
||||
*
|
||||
* Images are fetched from MLS Grid and cached locally on first request.
|
||||
* Per MLS Grid rules, images must be served from our own server.
|
||||
*
|
||||
* @param string $listing_key The listing key
|
||||
* @param bool $fetch_if_missing Whether to fetch from MLS Grid if not cached (default: true)
|
||||
* @return string|null Image URL or null
|
||||
*/
|
||||
function mls_get_property_image($listing_key) {
|
||||
function mls_get_property_image($listing_key, $fetch_if_missing = true) {
|
||||
$plugin = mls_plugin();
|
||||
if (!$plugin->get_query()) {
|
||||
if (!$plugin->get_media_handler()) {
|
||||
return null;
|
||||
}
|
||||
return $plugin->get_query()->get_primary_image($listing_key);
|
||||
return $plugin->get_media_handler()->get_primary_image($listing_key, $fetch_if_missing);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -324,3 +328,34 @@ function mls_get_property_count($args = array()) {
|
||||
}
|
||||
return $plugin->get_query()->get_count($args);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all images for a listing (on-demand fetching)
|
||||
*
|
||||
* Returns all media records with local_url populated where cached.
|
||||
* Can optionally fetch first N uncached images on-demand.
|
||||
*
|
||||
* @param string $listing_key The listing key
|
||||
* @param int $fetch_limit Max images to fetch on-demand (default: 1, 0 = none)
|
||||
* @return array Array of media objects
|
||||
*/
|
||||
function mls_get_property_images($listing_key, $fetch_limit = 1) {
|
||||
$plugin = mls_plugin();
|
||||
if (!$plugin->get_media_handler()) {
|
||||
return array();
|
||||
}
|
||||
return $plugin->get_media_handler()->get_listing_images($listing_key, $fetch_limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get media cache statistics
|
||||
*
|
||||
* @return array Stats with total_media, cached, uncached counts
|
||||
*/
|
||||
function mls_get_cache_stats() {
|
||||
$plugin = mls_plugin();
|
||||
if (!$plugin->get_media_handler()) {
|
||||
return array('total_media' => 0, 'cached' => 0, 'uncached' => 0);
|
||||
}
|
||||
return $plugin->get_media_handler()->get_cache_stats();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user