diff --git a/wp-content/plugins/mls-by-hansonxyz/cli/class-mls-cli.php b/wp-content/plugins/mls-by-hansonxyz/cli/class-mls-cli.php index a589b44b..20610c34 100644 --- a/wp-content/plugins/mls-by-hansonxyz/cli/class-mls-cli.php +++ b/wp-content/plugins/mls-by-hansonxyz/cli/class-mls-cli.php @@ -228,11 +228,18 @@ class MLS_CLI { * [--force] * : Force re-download of media (for media command) * + * [--quiet] + * : Suppress progress output + * + * [--verbose] + * : Show detailed output including API requests and responses + * * ## EXAMPLES * * wp mls sync full * wp mls sync full --dry-run --limit=10 * wp mls sync incremental + * wp mls sync incremental --verbose * wp mls sync media --limit=100 * wp mls sync resume --id=5 * @@ -242,16 +249,58 @@ class MLS_CLI { $type = isset($args[0]) ? $args[0] : 'incremental'; $dry_run = isset($assoc_args['dry-run']); $limit = isset($assoc_args['limit']) ? (int) $assoc_args['limit'] : null; + $quiet = isset($assoc_args['quiet']); + $verbose = isset($assoc_args['verbose']); $sync_engine = $this->plugin->get_sync_engine(); // Progress callback for CLI - $progress = null; - $progress_callback = function($stats) use (&$progress) { - if ($progress) { - $progress->tick(); - } - }; + $progress_callback = null; + if (!$quiet) { + $progress_callback = function($event, $data = array()) use ($verbose) { + if ($verbose) { + // Verbose mode: full line output + $this->output_verbose_event($event, $data); + } else { + // Compact mode: single character symbols + switch ($event) { + case 'property_created': + echo '.'; + break; + case 'property_updated': + echo '#'; + break; + case 'property_deleted': + echo 'x'; + break; + case 'property_skipped': + echo '-'; + break; + case 'property_error': + echo '!'; + break; + case 'media_downloaded': + echo 'P'; + break; + case 'media_skipped': + echo 'p'; + break; + case 'media_error': + echo 'E'; + break; + case 'page_complete': + echo '|'; + break; + case 'api_request': + // Silent in compact mode + break; + case 'api_response': + // Silent in compact mode + break; + } + } + }; + } switch ($type) { case 'full': @@ -259,8 +308,14 @@ class MLS_CLI { if ($dry_run) { WP_CLI::line('DRY RUN - No changes will be made'); } + if (!$quiet) { + $this->print_progress_legend($verbose); + } $result = $sync_engine->run_full_sync($dry_run, $limit, $progress_callback); + if (!$quiet) { + echo "\n"; + } $this->output_sync_result($result); break; @@ -269,17 +324,30 @@ class MLS_CLI { if ($dry_run) { WP_CLI::line('DRY RUN - No changes will be made'); } + if (!$quiet) { + $this->print_progress_legend($verbose); + } $result = $sync_engine->run_incremental_sync($dry_run, $progress_callback); + if (!$quiet) { + echo "\n"; + } $this->output_sync_result($result); break; case 'media': WP_CLI::line('Downloading pending media...'); + if (!$quiet) { + WP_CLI::line('Legend: P=downloaded p=skipped E=error'); + echo "\n"; + } $media_handler = $this->plugin->get_media_handler(); - $result = $media_handler->download_pending($limit ?: 100); + $result = $media_handler->download_pending($limit ?: 100, $progress_callback); + if (!$quiet) { + echo "\n"; + } WP_CLI::line(sprintf( 'Media download complete: %d success, %d failed out of %d total', $result['success'], @@ -293,6 +361,14 @@ class MLS_CLI { WP_CLI::success('No pending media to download.'); } else { WP_CLI::warning('Some media failed to download.'); + $missing_count = $media_handler->get_missing_count(); + if ($missing_count > 0) { + WP_CLI::line(sprintf( + 'Missing media log: %s (%d entries)', + $media_handler->get_missing_log_path(), + $missing_count + )); + } } break; @@ -303,8 +379,14 @@ class MLS_CLI { } WP_CLI::line("Resuming sync #{$sync_id}..."); + if (!$quiet) { + $this->print_progress_legend($verbose); + } $result = $sync_engine->resume_sync($sync_id, $progress_callback); + if (!$quiet) { + echo "\n"; + } $this->output_sync_result($result); break; @@ -313,6 +395,95 @@ class MLS_CLI { } } + /** + * Print progress legend + * + * @param bool $verbose Whether verbose mode is enabled + */ + private function print_progress_legend($verbose = false) { + if (!$verbose) { + WP_CLI::line('Legend: .=new #=updated x=deleted -=skipped !=error P=photo p=photo-skip E=photo-error |=page'); + echo "\n"; + } + } + + /** + * Output verbose event information + * + * @param string $event Event name + * @param array $data Event data + */ + private function output_verbose_event($event, $data) { + $timestamp = date('H:i:s'); + + switch ($event) { + case 'api_request': + WP_CLI::line("[{$timestamp}] API REQUEST: {$data['method']} {$data['url']}"); + if (!empty($data['params'])) { + WP_CLI::line(" Params: " . json_encode($data['params'])); + } + break; + + case 'api_response': + $status = $data['success'] ? 'OK' : 'ERROR'; + WP_CLI::line("[{$timestamp}] API RESPONSE: {$status} ({$data['status_code']}) - {$data['record_count']} records, {$data['response_time']}ms"); + if (!$data['success'] && !empty($data['error'])) { + WP_CLI::warning(" Error: {$data['error']}"); + } + break; + + case 'property_created': + $key = $data['listing_key'] ?? 'unknown'; + $address = $data['address'] ?? ''; + WP_CLI::line("[{$timestamp}] CREATED: {$key} {$address}"); + break; + + case 'property_updated': + $key = $data['listing_key'] ?? 'unknown'; + $address = $data['address'] ?? ''; + WP_CLI::line("[{$timestamp}] UPDATED: {$key} {$address}"); + break; + + case 'property_deleted': + $key = $data['listing_key'] ?? 'unknown'; + WP_CLI::line("[{$timestamp}] DELETED: {$key}"); + break; + + case 'property_skipped': + $key = $data['listing_key'] ?? 'unknown'; + WP_CLI::line("[{$timestamp}] SKIPPED: {$key} (dry-run)"); + break; + + case 'property_error': + $key = $data['listing_key'] ?? 'unknown'; + $error = $data['error'] ?? 'Unknown error'; + WP_CLI::warning("[{$timestamp}] ERROR: {$key} - {$error}"); + break; + + case 'media_downloaded': + $key = $data['media_key'] ?? 'unknown'; + $listing = $data['listing_key'] ?? ''; + WP_CLI::line("[{$timestamp}] MEDIA OK: {$key}" . ($listing ? " ({$listing})" : "")); + break; + + case 'media_skipped': + $key = $data['media_key'] ?? 'unknown'; + WP_CLI::line("[{$timestamp}] MEDIA SKIP: {$key} (already downloaded)"); + break; + + case 'media_error': + $key = $data['media_key'] ?? 'unknown'; + $error = $data['error'] ?? 'Download failed'; + WP_CLI::warning("[{$timestamp}] MEDIA ERROR: {$key} - {$error}"); + break; + + case 'page_complete': + $processed = $data['processed'] ?? 0; + WP_CLI::line("[{$timestamp}] PAGE COMPLETE: {$processed} records processed so far"); + break; + } + } + /** * Output sync result */ @@ -332,6 +503,17 @@ class MLS_CLI { $stats['deleted'], $stats['errors'] )); + + // Show missing media log info if there are failures + $media_handler = $this->plugin->get_media_handler(); + $missing_count = $media_handler->get_missing_count(); + if ($missing_count > 0) { + WP_CLI::line(sprintf( + 'Missing media log: %s (%d entries)', + $media_handler->get_missing_log_path(), + $missing_count + )); + } } /** @@ -380,7 +562,7 @@ class MLS_CLI { * ## OPTIONS * * - * : Action: clear, clear-listing, cleanup + * : Action: clear, clear-listing, cleanup, missing * * [--confirm] * : Confirm destructive operations @@ -388,11 +570,20 @@ class MLS_CLI { * [--listing=] * : Listing key for clear-listing * + * [--clear] + * : Clear the missing media log (for missing action) + * + * [--limit=] + * : Limit output lines (for missing action) + * * ## EXAMPLES * * wp mls cache clear --confirm * wp mls cache clear-listing --listing=NST123456 * wp mls cache cleanup + * wp mls cache missing + * wp mls cache missing --limit=20 + * wp mls cache missing --clear * * @subcommand cache */ @@ -421,6 +612,9 @@ class MLS_CLI { wp_mkdir_p($upload_dir); } + // Also clear missing log + $media_handler->clear_missing_log(); + WP_CLI::success('Cache cleared successfully.'); break; @@ -449,8 +643,44 @@ class MLS_CLI { WP_CLI::success("Cleaned up {$deleted} orphaned directories."); break; + case 'missing': + $media_handler = $this->plugin->get_media_handler(); + $log_file = $media_handler->get_missing_log_path(); + + if (isset($assoc_args['clear'])) { + $media_handler->clear_missing_log(); + WP_CLI::success('Missing media log cleared.'); + break; + } + + if (!file_exists($log_file)) { + WP_CLI::success('No missing media logged.'); + break; + } + + $limit = isset($assoc_args['limit']) ? (int) $assoc_args['limit'] : null; + $lines = file($log_file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + $total = count($lines); + + WP_CLI::line(sprintf('Missing media log: %s', $log_file)); + WP_CLI::line(sprintf('Total entries: %d', $total)); + WP_CLI::line(''); + WP_CLI::line('Format: [timestamp] listing_key | media_key | error | url'); + WP_CLI::line(str_repeat('-', 80)); + + $display = $limit ? array_slice($lines, 0, $limit) : $lines; + foreach ($display as $line) { + WP_CLI::line($line); + } + + if ($limit && $total > $limit) { + WP_CLI::line(''); + WP_CLI::line(sprintf('... and %d more entries', $total - $limit)); + } + break; + default: - WP_CLI::error("Unknown action: {$action}. Use 'clear', 'clear-listing', or 'cleanup'."); + WP_CLI::error("Unknown action: {$action}. Use 'clear', 'clear-listing', 'cleanup', or 'missing'."); } } diff --git a/wp-content/plugins/mls-by-hansonxyz/docs/CLAUDE.md b/wp-content/plugins/mls-by-hansonxyz/docs/CLAUDE.md index 2899d307..baed40d1 100644 --- a/wp-content/plugins/mls-by-hansonxyz/docs/CLAUDE.md +++ b/wp-content/plugins/mls-by-hansonxyz/docs/CLAUDE.md @@ -61,10 +61,10 @@ wp mls test auth wp mls status wp mls status rate-limits -# Run sync -wp mls sync full [--dry-run] [--limit=N] -wp mls sync incremental [--dry-run] -wp mls sync media [--limit=N] +# Run sync (use --verbose for detailed output) +wp mls sync full [--dry-run] [--limit=N] [--verbose] +wp mls sync incremental [--dry-run] [--verbose] +wp mls sync media [--limit=N] [--verbose] wp mls sync resume --id= # Statistics @@ -73,8 +73,33 @@ wp mls stats # Cache management wp mls cache clear --confirm wp mls cache cleanup +wp mls cache missing # View failed media downloads +wp mls cache missing --limit=20 # View first 20 entries +wp mls cache missing --clear # Clear the log ``` +### Progress Output + +Without --verbose (compact mode): +- `.` = new property created +- `#` = property updated +- `x` = property deleted +- `-` = skipped (dry-run) +- `P` = photo downloaded +- `p` = photo skipped (already exists) +- `E` = photo error +- `|` = page complete + +With --verbose: Full timestamped output showing API requests, responses, and individual item status. + +### Missing Media Log + +Failed media downloads are logged to: `wp-content/uploads/mls-missing-media.log` + +Format: `[timestamp] listing_key | media_key | error | url` + +Media downloads use exponential backoff (1s, 2s, 4s, 8s, 16s) for rate limit (429) and server errors (5xx). + ### Public API Functions Available for themes/plugins: diff --git a/wp-content/plugins/mls-by-hansonxyz/docs/USAGE.md b/wp-content/plugins/mls-by-hansonxyz/docs/USAGE.md index 8bdbbcf8..654ffb19 100644 --- a/wp-content/plugins/mls-by-hansonxyz/docs/USAGE.md +++ b/wp-content/plugins/mls-by-hansonxyz/docs/USAGE.md @@ -56,17 +56,53 @@ wp mls sync incremental # Download pending media wp mls sync media + +# Use --verbose for detailed output +wp mls sync full --verbose +wp mls sync incremental --verbose ``` -### Via Cron +#### Progress Indicators -Add to your system crontab for scheduled sync: +During sync, you'll see progress characters: +- `.` = new property +- `#` = updated property +- `x` = deleted property +- `P` = photo downloaded +- `p` = photo skipped (exists) +- `E` = photo error +- `|` = page complete + +Use `--verbose` for detailed timestamped output showing API requests and individual items. + +### Via Unix Cron + +Add to your system crontab (`crontab -e`) for scheduled sync: ```bash -# Run incremental sync every hour -0 * * * * cd /var/www/html && wp mls sync incremental --allow-root +# Incremental sync every hour (recommended for production) +0 * * * * cd /var/www/html && wp mls sync incremental --allow-root >> /var/log/mls-sync.log 2>&1 + +# Or every 30 minutes for more frequent updates +*/30 * * * * cd /var/www/html && wp mls sync incremental --allow-root >> /var/log/mls-sync.log 2>&1 + +# Full sync weekly (Sunday at 3am) to catch any missed records +0 3 * * 0 cd /var/www/html && wp mls sync full --allow-root >> /var/log/mls-sync.log 2>&1 + +# Download any pending media every 15 minutes +*/15 * * * * cd /var/www/html && wp mls sync media --limit=50 --allow-root >> /var/log/mls-sync.log 2>&1 ``` +**Important Notes:** +- Use `--allow-root` when running as root user +- Redirect output to a log file for debugging +- MLS Grid requires refresh at least every 12 hours per IDX rules +- The plugin handles rate limits automatically (waits if approaching limits) + +### Via WP-Cron (Alternative) + +Enable auto-sync in Settings > MLS Settings to use WordPress's built-in cron system. This runs on page loads rather than true system cron, so may be less reliable for high-frequency syncs. + ## Checking Status ### Via Admin @@ -237,6 +273,25 @@ wp mls cache clear --confirm This removes all synced data but keeps settings. +### Missing Media Log + +Failed media downloads are logged for review: + +```bash +# View missing media log +wp mls cache missing + +# View first 20 entries +wp mls cache missing --limit=20 + +# Clear the log +wp mls cache missing --clear +``` + +Log location: `wp-content/uploads/mls-missing-media.log` + +The log shows listing key, media key, error type, and original URL for each failed download. Media downloads automatically retry with exponential backoff (up to 5 attempts) for rate limit and server errors. + ## Support For plugin issues: Check logs at Settings > MLS Settings diff --git a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php index 6daecdd9..4512a945 100644 --- a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php +++ b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php @@ -72,8 +72,9 @@ class MLS_Media_Handler { * @param string $listing_key Listing key * @param array $media_array Media array from API * @param bool $force Force re-download all media + * @param callable|null $progress_callback Callback for progress updates */ - public function sync_property_media($listing_key, $media_array, $force = false) { + public function sync_property_media($listing_key, $media_array, $force = false, $progress_callback = null) { global $wpdb; if (empty($media_array)) { @@ -122,15 +123,36 @@ class MLS_Media_Handler { // Check if we need to re-download if ($force || $this->needs_download($existing, $media)) { - $this->download_media($existing->id); + $result = $this->download_media($existing->id); + if ($progress_callback) { + if ($result) { + call_user_func($progress_callback, 'media_downloaded', array('media_key' => $media_key)); + } else { + $error = $this->get_last_download_error($existing->id); + call_user_func($progress_callback, 'media_error', array('media_key' => $media_key, 'error' => $error)); + } + } + } else { + if ($progress_callback) { + call_user_func($progress_callback, 'media_skipped', array('media_key' => $media_key)); + } } } else { // Insert new record $data['created_at'] = current_time('mysql'); $wpdb->insert($this->db->media_table(), $data); + $new_id = $wpdb->insert_id; // Queue download - $this->download_media($wpdb->insert_id); + $result = $this->download_media($new_id); + if ($progress_callback) { + if ($result) { + call_user_func($progress_callback, 'media_downloaded', array('media_key' => $media_key)); + } else { + $error = $this->get_last_download_error($new_id); + call_user_func($progress_callback, 'media_error', array('media_key' => $media_key, 'error' => $error)); + } + } } } @@ -160,6 +182,74 @@ class MLS_Media_Handler { } } + /** + * Get the last download error for a media record + * + * @param int $media_id Media ID + * @return string|null Error message + */ + private function get_last_download_error($media_id) { + global $wpdb; + return $wpdb->get_var($wpdb->prepare( + "SELECT download_error FROM {$this->db->media_table()} WHERE id = %d", + $media_id + )); + } + + /** + * Get the path to the missing media log file + * + * @return string File path + */ + public function get_missing_log_path() { + $upload_dir = wp_upload_dir(); + return $upload_dir['basedir'] . '/mls-missing-media.log'; + } + + /** + * Log a failed media download to the missing media log file + * + * @param object $media Media record + * @param string $error Error message + */ + private function log_missing_media($media, $error) { + $log_file = $this->get_missing_log_path(); + $timestamp = date('Y-m-d H:i:s'); + $line = sprintf( + "[%s] %s | %s | %s | %s\n", + $timestamp, + $media->listing_key, + $media->media_key, + $error, + $media->media_url + ); + file_put_contents($log_file, $line, FILE_APPEND | LOCK_EX); + } + + /** + * Clear the missing media log file + */ + public function clear_missing_log() { + $log_file = $this->get_missing_log_path(); + if (file_exists($log_file)) { + unlink($log_file); + } + } + + /** + * Get missing media count from log file + * + * @return int Number of missing media entries + */ + public function get_missing_count() { + $log_file = $this->get_missing_log_path(); + if (!file_exists($log_file)) { + return 0; + } + $content = file_get_contents($log_file); + return substr_count($content, "\n"); + } + /** * Check if media needs to be downloaded * @@ -212,34 +302,87 @@ class MLS_Media_Handler { array('id' => $media_id) ); - // Download file - $response = wp_remote_get($media->media_url, array( - 'timeout' => 60, - 'stream' => false, - )); + // Download with exponential backoff for rate limits + $max_retries = 5; + $response = null; + $status_code = 0; + $base_delay = 1; // Start with 1 second - if (is_wp_error($response)) { - $this->logger->warning('Media download failed', array( - 'media_id' => $media_id, - 'error' => $response->get_error_message(), + for ($retry = 0; $retry < $max_retries; $retry++) { + // Exponential backoff: 1s, 2s, 4s, 8s, 16s + if ($retry > 0) { + $delay = $base_delay * pow(2, $retry - 1); + $this->logger->debug('Media download retry', array( + 'media_id' => $media_id, + 'retry' => $retry, + 'delay' => $delay, + )); + sleep($delay); + } + + $response = wp_remote_get($media->media_url, array( + 'timeout' => 60, + 'stream' => false, )); + if (is_wp_error($response)) { + $error_msg = $response->get_error_message(); + $this->logger->warning('Media download failed', array( + 'media_id' => $media_id, + 'error' => $error_msg, + 'retry' => $retry, + )); + + if ($retry === $max_retries - 1) { + $wpdb->update( + $this->db->media_table(), + array('download_error' => $error_msg), + array('id' => $media_id) + ); + $this->log_missing_media($media, $error_msg); + return false; + } + continue; + } + + $status_code = wp_remote_retrieve_response_code($response); + + // Success + if ($status_code === 200) { + break; + } + + // Retryable errors: 429 (rate limit), 500, 502, 503, 504 (server errors) + $retryable = in_array($status_code, array(429, 500, 502, 503, 504)); + + if ($retryable && $retry < $max_retries - 1) { + $this->logger->debug('Media download retryable error', array( + 'media_id' => $media_id, + 'status_code' => $status_code, + 'retry' => $retry, + )); + continue; + } + + // Non-retryable or exhausted retries - record and fail + $error_msg = "HTTP {$status_code}"; $wpdb->update( $this->db->media_table(), - array('download_error' => $response->get_error_message()), + array('download_error' => $error_msg), array('id' => $media_id) ); - + $this->log_missing_media($media, $error_msg); return false; } - $status_code = wp_remote_retrieve_response_code($response); if ($status_code !== 200) { + $error_msg = "HTTP {$status_code}"; $wpdb->update( $this->db->media_table(), - array('download_error' => "HTTP {$status_code}"), + array('download_error' => $error_msg), array('id' => $media_id) ); + $this->log_missing_media($media, $error_msg); return false; } @@ -418,13 +561,14 @@ class MLS_Media_Handler { * Download pending media (for batch processing) * * @param int $limit Max media to download + * @param callable|null $progress_callback Callback for progress updates * @return array Stats */ - public function download_pending($limit = 100) { + public function download_pending($limit = 100, $progress_callback = null) { global $wpdb; $pending = $wpdb->get_results($wpdb->prepare( - "SELECT id FROM {$this->db->media_table()} + "SELECT id, media_key FROM {$this->db->media_table()} WHERE local_path IS NULL AND media_url IS NOT NULL AND download_attempts < 3 LIMIT %d", @@ -440,8 +584,14 @@ class MLS_Media_Handler { foreach ($pending as $media) { if ($this->download_media($media->id)) { $stats['success']++; + if ($progress_callback) { + call_user_func($progress_callback, 'media_downloaded', array('media_key' => $media->media_key)); + } } else { $stats['failed']++; + if ($progress_callback) { + call_user_func($progress_callback, 'media_error', array('media_key' => $media->media_key)); + } } } diff --git a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php index e49b7539..4532c0f8 100644 --- a/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php +++ b/wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php @@ -88,6 +88,9 @@ class MLS_Sync_Engine { public function run_full_sync($dry_run = false, $limit = null, $progress_callback = null) { $this->logger->info('Starting full sync', array('dry_run' => $dry_run, 'limit' => $limit)); + // Store progress callback for use in process_property + $this->progress_callback = $progress_callback; + // Create sync state record if (!$dry_run) { $this->sync_state_id = $this->create_sync_state(self::TYPE_FULL); @@ -104,12 +107,35 @@ class MLS_Sync_Engine { try { // Get first page of properties with media + $start_time = microtime(true); + $this->emit_progress('api_request', array( + 'method' => 'GET', + 'url' => 'Property', + 'params' => array('type' => 'full_sync', 'limit' => $limit), + )); + $response = $this->api_client->get_properties_for_sync(null, 'Media', $limit ? min($limit, 1000) : null); + $elapsed = round((microtime(true) - $start_time) * 1000); if (is_wp_error($response)) { + $this->emit_progress('api_response', array( + 'success' => false, + 'status_code' => 0, + 'error' => $response->get_error_message(), + 'record_count' => 0, + 'response_time' => $elapsed, + )); throw new Exception($response->get_error_message()); } + $this->emit_progress('api_response', array( + 'success' => true, + 'status_code' => 200, + 'record_count' => isset($response['value']) ? count($response['value']) : 0, + 'response_time' => $elapsed, + 'has_more' => isset($response['@odata.nextLink']), + )); + // Process pages $continue = true; while ($continue && isset($response['value'])) { @@ -120,12 +146,11 @@ class MLS_Sync_Engine { } $this->process_property($property, $dry_run); - - if ($progress_callback) { - call_user_func($progress_callback, $this->stats); - } } + // Emit page complete event + $this->emit_progress('page_complete', array('processed' => $this->stats['processed'])); + // Check for next page if ($continue && isset($response['@odata.nextLink'])) { // Save progress @@ -138,11 +163,34 @@ class MLS_Sync_Engine { )); } + $start_time = microtime(true); + $this->emit_progress('api_request', array( + 'method' => 'GET', + 'url' => 'Property (next page)', + 'params' => array('page' => 'next'), + )); + $response = $this->api_client->get_next_page($response['@odata.nextLink']); + $elapsed = round((microtime(true) - $start_time) * 1000); if (is_wp_error($response)) { + $this->emit_progress('api_response', array( + 'success' => false, + 'status_code' => 0, + 'error' => $response->get_error_message(), + 'record_count' => 0, + 'response_time' => $elapsed, + )); throw new Exception($response->get_error_message()); } + + $this->emit_progress('api_response', array( + 'success' => true, + 'status_code' => 200, + 'record_count' => isset($response['value']) ? count($response['value']) : 0, + 'response_time' => $elapsed, + 'has_more' => isset($response['@odata.nextLink']), + )); } else { $continue = false; } @@ -211,6 +259,9 @@ class MLS_Sync_Engine { 'dry_run' => $dry_run, )); + // Store progress callback for use in process_property + $this->progress_callback = $progress_callback; + if (!$dry_run) { $this->sync_state_id = $this->create_sync_state(self::TYPE_INCREMENTAL); $this->logger->set_sync_state($this->sync_state_id); @@ -226,29 +277,74 @@ class MLS_Sync_Engine { try { // Get modified properties (including those marked for deletion) + $start_time = microtime(true); + $this->emit_progress('api_request', array( + 'method' => 'GET', + 'url' => 'Property', + 'params' => array('type' => 'incremental', 'since' => $last_timestamp), + )); + $response = $this->api_client->get_properties_since($last_timestamp, 'Media'); + $elapsed = round((microtime(true) - $start_time) * 1000); if (is_wp_error($response)) { + $this->emit_progress('api_response', array( + 'success' => false, + 'status_code' => 0, + 'error' => $response->get_error_message(), + 'record_count' => 0, + 'response_time' => $elapsed, + )); throw new Exception($response->get_error_message()); } + $this->emit_progress('api_response', array( + 'success' => true, + 'status_code' => 200, + 'record_count' => isset($response['value']) ? count($response['value']) : 0, + 'response_time' => $elapsed, + 'has_more' => isset($response['@odata.nextLink']), + )); + // Process pages while (isset($response['value'])) { foreach ($response['value'] as $property) { $this->process_property($property, $dry_run); - - if ($progress_callback) { - call_user_func($progress_callback, $this->stats); - } } + // Emit page complete event + $this->emit_progress('page_complete', array('processed' => $this->stats['processed'])); + // Check for next page if (isset($response['@odata.nextLink'])) { + $start_time = microtime(true); + $this->emit_progress('api_request', array( + 'method' => 'GET', + 'url' => 'Property (next page)', + 'params' => array('page' => 'next'), + )); + $response = $this->api_client->get_next_page($response['@odata.nextLink']); + $elapsed = round((microtime(true) - $start_time) * 1000); if (is_wp_error($response)) { + $this->emit_progress('api_response', array( + 'success' => false, + 'status_code' => 0, + 'error' => $response->get_error_message(), + 'record_count' => 0, + 'response_time' => $elapsed, + )); throw new Exception($response->get_error_message()); } + + $this->emit_progress('api_response', array( + 'success' => true, + 'status_code' => 200, + 'record_count' => isset($response['value']) ? count($response['value']) : 0, + 'response_time' => $elapsed, + 'has_more' => isset($response['@odata.nextLink']), + )); } else { break; } @@ -324,6 +420,7 @@ class MLS_Sync_Engine { } $this->sync_state_id = $sync_state_id; + $this->progress_callback = $progress_callback; $this->logger->set_sync_state($sync_state_id); $this->logger->info('Resuming sync', array('sync_state_id' => $sync_state_id)); @@ -359,12 +456,11 @@ class MLS_Sync_Engine { while (isset($response['value'])) { foreach ($response['value'] as $property) { $this->process_property($property, false); - - if ($progress_callback) { - call_user_func($progress_callback, $this->stats); - } } + // Emit page complete event + $this->emit_progress('page_complete', array('processed' => $this->stats['processed'])); + if (isset($response['@odata.nextLink'])) { $this->update_sync_state(array( 'last_next_link' => $response['@odata.nextLink'], @@ -413,6 +509,11 @@ class MLS_Sync_Engine { ); } + /** + * Progress callback reference + */ + private $progress_callback = null; + /** * Process a single property record * @@ -439,6 +540,7 @@ class MLS_Sync_Engine { $this->delete_property($listing_key); } $this->stats['deleted']++; + $this->emit_progress('property_deleted', array('listing_key' => $listing_key)); return; } @@ -454,8 +556,10 @@ class MLS_Sync_Engine { if ($dry_run) { if ($existing) { $this->stats['updated']++; + $this->emit_progress('property_skipped', array('listing_key' => $listing_key)); } else { $this->stats['created']++; + $this->emit_progress('property_skipped', array('listing_key' => $listing_key)); } return; } @@ -468,17 +572,31 @@ class MLS_Sync_Engine { array('listing_key' => $listing_key) ); $this->stats['updated']++; + $this->emit_progress('property_updated', array('listing_key' => $listing_key)); } else { // Insert new $data['listing_key'] = $listing_key; $data['created_at'] = current_time('mysql'); $wpdb->insert($this->db->properties_table(), $data); $this->stats['created']++; + $this->emit_progress('property_created', array('listing_key' => $listing_key)); } // Process media if present if (isset($property['Media']) && is_array($property['Media'])) { - $this->media_handler->sync_property_media($listing_key, $property['Media']); + $this->media_handler->sync_property_media($listing_key, $property['Media'], false, $this->progress_callback); + } + } + + /** + * Emit progress event + * + * @param string $event Event name + * @param array $data Event data + */ + private function emit_progress($event, $data = array()) { + if ($this->progress_callback) { + call_user_func($this->progress_callback, $event, $data); } }