brianhansonxyz/homeproz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add verbose mode, progress indicators, and missing media log

Browse Source 
- Add --verbose flag to sync commands for detailed API request/response output
- Add progress indicators (.=#xPpE|) for compact sync output
- Implement exponential backoff (1s, 2s, 4s, 8s, 16s) for media downloads
- Log failed media downloads to wp-content/uploads/mls-missing-media.log
- Add 'wp mls cache missing' command to view/clear the log
- Retry on rate limit (429) and server errors (5xx)
- Update documentation with new features

Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Hanson.xyz Dev
2025-12-14 22:20:41 -06:00
parent 6556479417
commit 5e4ebfb99e
5 changed files with 626 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wp-content/plugins/mls-by-hansonxyz/cli/class-mls-cli.php
+239 -9
View File
@@ -228,11 +228,18 @@ class MLS_CLI {
* [--force]
* : Force re-download of media (for media command)
*
* [--quiet]
* : Suppress progress output
*
* [--verbose]
* : Show detailed output including API requests and responses
*
* ## EXAMPLES
*
* wp mls sync full
* wp mls sync full --dry-run --limit=10
* wp mls sync incremental
* wp mls sync incremental --verbose
* wp mls sync media --limit=100
* wp mls sync resume --id=5
*
@@ -242,16 +249,58 @@ class MLS_CLI {
$type = isset($args[0]) ? $args[0] : 'incremental';
$dry_run = isset($assoc_args['dry-run']);
$limit = isset($assoc_args['limit']) ? (int) $assoc_args['limit'] : null;
$quiet = isset($assoc_args['quiet']);
$verbose = isset($assoc_args['verbose']);
$sync_engine = $this->plugin->get_sync_engine();
// Progress callback for CLI
$progress = null;
$progress_callback = function($stats) use (&$progress) {
if ($progress) {
$progress->tick();
}
};
$progress_callback = null;
if (!$quiet) {
$progress_callback = function($event, $data = array()) use ($verbose) {
if ($verbose) {
// Verbose mode: full line output
$this->output_verbose_event($event, $data);
} else {
// Compact mode: single character symbols
switch ($event) {
case 'property_created':
echo '.';
break;
case 'property_updated':
echo '#';
break;
case 'property_deleted':
echo 'x';
break;
case 'property_skipped':
echo '-';
break;
case 'property_error':
echo '!';
break;
case 'media_downloaded':
echo 'P';
break;
case 'media_skipped':
echo 'p';
break;
case 'media_error':
echo 'E';
break;
case 'page_complete':
echo '|';
break;
case 'api_request':
// Silent in compact mode
break;
case 'api_response':
// Silent in compact mode
break;
}
}
};
}
switch ($type) {
case 'full':
@@ -259,8 +308,14 @@ class MLS_CLI {
if ($dry_run) {
WP_CLI::line('DRY RUN - No changes will be made');
}
if (!$quiet) {
$this->print_progress_legend($verbose);
}
$result = $sync_engine->run_full_sync($dry_run, $limit, $progress_callback);
if (!$quiet) {
echo "\n";
}
$this->output_sync_result($result);
break;
@@ -269,17 +324,30 @@ class MLS_CLI {
if ($dry_run) {
WP_CLI::line('DRY RUN - No changes will be made');
}
if (!$quiet) {
$this->print_progress_legend($verbose);
}
$result = $sync_engine->run_incremental_sync($dry_run, $progress_callback);
if (!$quiet) {
echo "\n";
}
$this->output_sync_result($result);
break;
case 'media':
WP_CLI::line('Downloading pending media...');
if (!$quiet) {
WP_CLI::line('Legend: P=downloaded p=skipped E=error');
echo "\n";
}
$media_handler = $this->plugin->get_media_handler();
$result = $media_handler->download_pending($limit ?: 100);
$result = $media_handler->download_pending($limit ?: 100, $progress_callback);
if (!$quiet) {
echo "\n";
}
WP_CLI::line(sprintf(
'Media download complete: %d success, %d failed out of %d total',
$result['success'],
@@ -293,6 +361,14 @@ class MLS_CLI {
WP_CLI::success('No pending media to download.');
} else {
WP_CLI::warning('Some media failed to download.');
$missing_count = $media_handler->get_missing_count();
if ($missing_count > 0) {
WP_CLI::line(sprintf(
'Missing media log: %s (%d entries)',
$media_handler->get_missing_log_path(),
$missing_count
));
}
}
break;
@@ -303,8 +379,14 @@ class MLS_CLI {
}
WP_CLI::line("Resuming sync #{$sync_id}...");
if (!$quiet) {
$this->print_progress_legend($verbose);
}
$result = $sync_engine->resume_sync($sync_id, $progress_callback);
if (!$quiet) {
echo "\n";
}
$this->output_sync_result($result);
break;
@@ -313,6 +395,95 @@ class MLS_CLI {
}
}
/**
* Print progress legend
*
* @param bool $verbose Whether verbose mode is enabled
*/
private function print_progress_legend($verbose = false) {
if (!$verbose) {
WP_CLI::line('Legend: .=new #=updated x=deleted -=skipped !=error P=photo p=photo-skip E=photo-error |=page');
echo "\n";
}
}
/**
* Output verbose event information
*
* @param string $event Event name
* @param array $data Event data
*/
private function output_verbose_event($event, $data) {
$timestamp = date('H:i:s');
switch ($event) {
case 'api_request':
WP_CLI::line("[{$timestamp}] API REQUEST: {$data['method']} {$data['url']}");
if (!empty($data['params'])) {
WP_CLI::line(" Params: " . json_encode($data['params']));
}
break;
case 'api_response':
$status = $data['success'] ? 'OK' : 'ERROR';
WP_CLI::line("[{$timestamp}] API RESPONSE: {$status} ({$data['status_code']}) - {$data['record_count']} records, {$data['response_time']}ms");
if (!$data['success'] && !empty($data['error'])) {
WP_CLI::warning(" Error: {$data['error']}");
}
break;
case 'property_created':
$key = $data['listing_key'] ?? 'unknown';
$address = $data['address'] ?? '';
WP_CLI::line("[{$timestamp}] CREATED: {$key} {$address}");
break;
case 'property_updated':
$key = $data['listing_key'] ?? 'unknown';
$address = $data['address'] ?? '';
WP_CLI::line("[{$timestamp}] UPDATED: {$key} {$address}");
break;
case 'property_deleted':
$key = $data['listing_key'] ?? 'unknown';
WP_CLI::line("[{$timestamp}] DELETED: {$key}");
break;
case 'property_skipped':
$key = $data['listing_key'] ?? 'unknown';
WP_CLI::line("[{$timestamp}] SKIPPED: {$key} (dry-run)");
break;
case 'property_error':
$key = $data['listing_key'] ?? 'unknown';
$error = $data['error'] ?? 'Unknown error';
WP_CLI::warning("[{$timestamp}] ERROR: {$key} - {$error}");
break;
case 'media_downloaded':
$key = $data['media_key'] ?? 'unknown';
$listing = $data['listing_key'] ?? '';
WP_CLI::line("[{$timestamp}] MEDIA OK: {$key}" . ($listing ? " ({$listing})" : ""));
break;
case 'media_skipped':
$key = $data['media_key'] ?? 'unknown';
WP_CLI::line("[{$timestamp}] MEDIA SKIP: {$key} (already downloaded)");
break;
case 'media_error':
$key = $data['media_key'] ?? 'unknown';
$error = $data['error'] ?? 'Download failed';
WP_CLI::warning("[{$timestamp}] MEDIA ERROR: {$key} - {$error}");
break;
case 'page_complete':
$processed = $data['processed'] ?? 0;
WP_CLI::line("[{$timestamp}] PAGE COMPLETE: {$processed} records processed so far");
break;
}
}
/**
* Output sync result
*/
@@ -332,6 +503,17 @@ class MLS_CLI {
$stats['deleted'],
$stats['errors']
));
// Show missing media log info if there are failures
$media_handler = $this->plugin->get_media_handler();
$missing_count = $media_handler->get_missing_count();
if ($missing_count > 0) {
WP_CLI::line(sprintf(
'Missing media log: %s (%d entries)',
$media_handler->get_missing_log_path(),
$missing_count
));
}
}
/**
@@ -380,7 +562,7 @@ class MLS_CLI {
* ## OPTIONS
*
* <action>
* : Action: clear, clear-listing, cleanup
* : Action: clear, clear-listing, cleanup, missing
*
* [--confirm]
* : Confirm destructive operations
@@ -388,11 +570,20 @@ class MLS_CLI {
* [--listing=<key>]
* : Listing key for clear-listing
*
* [--clear]
* : Clear the missing media log (for missing action)
*
* [--limit=<n>]
* : Limit output lines (for missing action)
*
* ## EXAMPLES
*
* wp mls cache clear --confirm
* wp mls cache clear-listing --listing=NST123456
* wp mls cache cleanup
* wp mls cache missing
* wp mls cache missing --limit=20
* wp mls cache missing --clear
*
* @subcommand cache
*/
@@ -421,6 +612,9 @@ class MLS_CLI {
wp_mkdir_p($upload_dir);
}
// Also clear missing log
$media_handler->clear_missing_log();
WP_CLI::success('Cache cleared successfully.');
break;
@@ -449,8 +643,44 @@ class MLS_CLI {
WP_CLI::success("Cleaned up {$deleted} orphaned directories.");
break;
case 'missing':
$media_handler = $this->plugin->get_media_handler();
$log_file = $media_handler->get_missing_log_path();
if (isset($assoc_args['clear'])) {
$media_handler->clear_missing_log();
WP_CLI::success('Missing media log cleared.');
break;
}
if (!file_exists($log_file)) {
WP_CLI::success('No missing media logged.');
break;
}
$limit = isset($assoc_args['limit']) ? (int) $assoc_args['limit'] : null;
$lines = file($log_file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
$total = count($lines);
WP_CLI::line(sprintf('Missing media log: %s', $log_file));
WP_CLI::line(sprintf('Total entries: %d', $total));
WP_CLI::line('');
WP_CLI::line('Format: [timestamp] listing_key | media_key | error | url');
WP_CLI::line(str_repeat('-', 80));
$display = $limit ? array_slice($lines, 0, $limit) : $lines;
foreach ($display as $line) {
WP_CLI::line($line);
}
if ($limit && $total > $limit) {
WP_CLI::line('');
WP_CLI::line(sprintf('... and %d more entries', $total - $limit));
}
break;
default:
WP_CLI::error("Unknown action: {$action}. Use 'clear', 'clear-listing', or 'cleanup'.");
WP_CLI::error("Unknown action: {$action}. Use 'clear', 'clear-listing', 'cleanup', or 'missing'.");
}
}
wp-content/plugins/mls-by-hansonxyz/docs/CLAUDE.md
+29 -4
View File
@@ -61,10 +61,10 @@ wp mls test auth
wp mls status
wp mls status rate-limits
# Run sync
wp mls sync full [--dry-run] [--limit=N]
wp mls sync incremental [--dry-run]
wp mls sync media [--limit=N]
# Run sync (use --verbose for detailed output)
wp mls sync full [--dry-run] [--limit=N] [--verbose]
wp mls sync incremental [--dry-run] [--verbose]
wp mls sync media [--limit=N] [--verbose]
wp mls sync resume --id=<sync_id>
# Statistics
@@ -73,8 +73,33 @@ wp mls stats
# Cache management
wp mls cache clear --confirm
wp mls cache cleanup
wp mls cache missing # View failed media downloads
wp mls cache missing --limit=20 # View first 20 entries
wp mls cache missing --clear # Clear the log
```
### Progress Output
Without --verbose (compact mode):
- `.` = new property created
- `#` = property updated
- `x` = property deleted
- `-` = skipped (dry-run)
- `P` = photo downloaded
- `p` = photo skipped (already exists)
- `E` = photo error
- `|` = page complete
With --verbose: Full timestamped output showing API requests, responses, and individual item status.
### Missing Media Log
Failed media downloads are logged to: `wp-content/uploads/mls-missing-media.log`
Format: `[timestamp] listing_key | media_key | error | url`
Media downloads use exponential backoff (1s, 2s, 4s, 8s, 16s) for rate limit (429) and server errors (5xx).
### Public API Functions
Available for themes/plugins:
wp-content/plugins/mls-by-hansonxyz/docs/USAGE.md
+59 -4
View File
@@ -56,17 +56,53 @@ wp mls sync incremental
# Download pending media
wp mls sync media
# Use --verbose for detailed output
wp mls sync full --verbose
wp mls sync incremental --verbose
```
### Via Cron
#### Progress Indicators
Add to your system crontab for scheduled sync:
During sync, you'll see progress characters:
- `.` = new property
- `#` = updated property
- `x` = deleted property
- `P` = photo downloaded
- `p` = photo skipped (exists)
- `E` = photo error
- `|` = page complete
Use `--verbose` for detailed timestamped output showing API requests and individual items.
### Via Unix Cron
Add to your system crontab (`crontab -e`) for scheduled sync:
```bash
# Run incremental sync every hour
0 * * * * cd /var/www/html && wp mls sync incremental --allow-root
# Incremental sync every hour (recommended for production)
0 * * * * cd /var/www/html && wp mls sync incremental --allow-root >> /var/log/mls-sync.log 2>&1
# Or every 30 minutes for more frequent updates
*/30 * * * * cd /var/www/html && wp mls sync incremental --allow-root >> /var/log/mls-sync.log 2>&1
# Full sync weekly (Sunday at 3am) to catch any missed records
0 3 * * 0 cd /var/www/html && wp mls sync full --allow-root >> /var/log/mls-sync.log 2>&1
# Download any pending media every 15 minutes
*/15 * * * * cd /var/www/html && wp mls sync media --limit=50 --allow-root >> /var/log/mls-sync.log 2>&1
```
**Important Notes:**
- Use `--allow-root` when running as root user
- Redirect output to a log file for debugging
- MLS Grid requires refresh at least every 12 hours per IDX rules
- The plugin handles rate limits automatically (waits if approaching limits)
### Via WP-Cron (Alternative)
Enable auto-sync in Settings > MLS Settings to use WordPress's built-in cron system. This runs on page loads rather than true system cron, so may be less reliable for high-frequency syncs.
## Checking Status
### Via Admin
@@ -237,6 +273,25 @@ wp mls cache clear --confirm
This removes all synced data but keeps settings.
### Missing Media Log
Failed media downloads are logged for review:
```bash
# View missing media log
wp mls cache missing
# View first 20 entries
wp mls cache missing --limit=20
# Clear the log
wp mls cache missing --clear
```
Log location: `wp-content/uploads/mls-missing-media.log`
The log shows listing key, media key, error type, and original URL for each failed download. Media downloads automatically retry with exponential backoff (up to 5 attempts) for rate limit and server errors.
## Support
For plugin issues: Check logs at Settings > MLS Settings
wp-content/plugins/mls-by-hansonxyz/includes/class-mls-media-handler.php
+168 -18
View File
@@ -72,8 +72,9 @@ class MLS_Media_Handler {
* @param string $listing_key Listing key
* @param array $media_array Media array from API
* @param bool $force Force re-download all media
* @param callable|null $progress_callback Callback for progress updates
*/
public function sync_property_media($listing_key, $media_array, $force = false) {
public function sync_property_media($listing_key, $media_array, $force = false, $progress_callback = null) {
global $wpdb;
if (empty($media_array)) {
@@ -122,15 +123,36 @@ class MLS_Media_Handler {
// Check if we need to re-download
if ($force || $this->needs_download($existing, $media)) {
$this->download_media($existing->id);
$result = $this->download_media($existing->id);
if ($progress_callback) {
if ($result) {
call_user_func($progress_callback, 'media_downloaded', array('media_key' => $media_key));
} else {
$error = $this->get_last_download_error($existing->id);
call_user_func($progress_callback, 'media_error', array('media_key' => $media_key, 'error' => $error));
}
}
} else {
if ($progress_callback) {
call_user_func($progress_callback, 'media_skipped', array('media_key' => $media_key));
}
}
} else {
// Insert new record
$data['created_at'] = current_time('mysql');
$wpdb->insert($this->db->media_table(), $data);
$new_id = $wpdb->insert_id;
// Queue download
$this->download_media($wpdb->insert_id);
$result = $this->download_media($new_id);
if ($progress_callback) {
if ($result) {
call_user_func($progress_callback, 'media_downloaded', array('media_key' => $media_key));
} else {
$error = $this->get_last_download_error($new_id);
call_user_func($progress_callback, 'media_error', array('media_key' => $media_key, 'error' => $error));
}
}
}
}
@@ -160,6 +182,74 @@ class MLS_Media_Handler {
}
}
/**
* Get the last download error for a media record
*
* @param int $media_id Media ID
* @return string|null Error message
*/
private function get_last_download_error($media_id) {
global $wpdb;
return $wpdb->get_var($wpdb->prepare(
"SELECT download_error FROM {$this->db->media_table()} WHERE id = %d",
$media_id
));
}
/**
* Get the path to the missing media log file
*
* @return string File path
*/
public function get_missing_log_path() {
$upload_dir = wp_upload_dir();
return $upload_dir['basedir'] . '/mls-missing-media.log';
}
/**
* Log a failed media download to the missing media log file
*
* @param object $media Media record
* @param string $error Error message
*/
private function log_missing_media($media, $error) {
$log_file = $this->get_missing_log_path();
$timestamp = date('Y-m-d H:i:s');
$line = sprintf(
"[%s] %s | %s | %s | %s\n",
$timestamp,
$media->listing_key,
$media->media_key,
$error,
$media->media_url
);
file_put_contents($log_file, $line, FILE_APPEND | LOCK_EX);
}
/**
* Clear the missing media log file
*/
public function clear_missing_log() {
$log_file = $this->get_missing_log_path();
if (file_exists($log_file)) {
unlink($log_file);
}
}
/**
* Get missing media count from log file
*
* @return int Number of missing media entries
*/
public function get_missing_count() {
$log_file = $this->get_missing_log_path();
if (!file_exists($log_file)) {
return 0;
}
$content = file_get_contents($log_file);
return substr_count($content, "\n");
}
/**
* Check if media needs to be downloaded
*
@@ -212,34 +302,87 @@ class MLS_Media_Handler {
array('id' => $media_id)
);
// Download file
$response = wp_remote_get($media->media_url, array(
'timeout' => 60,
'stream' => false,
));
// Download with exponential backoff for rate limits
$max_retries = 5;
$response = null;
$status_code = 0;
$base_delay = 1; // Start with 1 second
if (is_wp_error($response)) {
$this->logger->warning('Media download failed', array(
'media_id' => $media_id,
'error' => $response->get_error_message(),
for ($retry = 0; $retry < $max_retries; $retry++) {
// Exponential backoff: 1s, 2s, 4s, 8s, 16s
if ($retry > 0) {
$delay = $base_delay * pow(2, $retry - 1);
$this->logger->debug('Media download retry', array(
'media_id' => $media_id,
'retry' => $retry,
'delay' => $delay,
));
sleep($delay);
}
$response = wp_remote_get($media->media_url, array(
'timeout' => 60,
'stream' => false,
));
if (is_wp_error($response)) {
$error_msg = $response->get_error_message();
$this->logger->warning('Media download failed', array(
'media_id' => $media_id,
'error' => $error_msg,
'retry' => $retry,
));
if ($retry === $max_retries - 1) {
$wpdb->update(
$this->db->media_table(),
array('download_error' => $error_msg),
array('id' => $media_id)
);
$this->log_missing_media($media, $error_msg);
return false;
}
continue;
}
$status_code = wp_remote_retrieve_response_code($response);
// Success
if ($status_code === 200) {
break;
}
// Retryable errors: 429 (rate limit), 500, 502, 503, 504 (server errors)
$retryable = in_array($status_code, array(429, 500, 502, 503, 504));
if ($retryable && $retry < $max_retries - 1) {
$this->logger->debug('Media download retryable error', array(
'media_id' => $media_id,
'status_code' => $status_code,
'retry' => $retry,
));
continue;
}
// Non-retryable or exhausted retries - record and fail
$error_msg = "HTTP {$status_code}";
$wpdb->update(
$this->db->media_table(),
array('download_error' => $response->get_error_message()),
array('download_error' => $error_msg),
array('id' => $media_id)
);
$this->log_missing_media($media, $error_msg);
return false;
}
$status_code = wp_remote_retrieve_response_code($response);
if ($status_code !== 200) {
$error_msg = "HTTP {$status_code}";
$wpdb->update(
$this->db->media_table(),
array('download_error' => "HTTP {$status_code}"),
array('download_error' => $error_msg),
array('id' => $media_id)
);
$this->log_missing_media($media, $error_msg);
return false;
}
@@ -418,13 +561,14 @@ class MLS_Media_Handler {
* Download pending media (for batch processing)
*
* @param int $limit Max media to download
* @param callable|null $progress_callback Callback for progress updates
* @return array Stats
*/
public function download_pending($limit = 100) {
public function download_pending($limit = 100, $progress_callback = null) {
global $wpdb;
$pending = $wpdb->get_results($wpdb->prepare(
"SELECT id FROM {$this->db->media_table()}
"SELECT id, media_key FROM {$this->db->media_table()}
WHERE local_path IS NULL AND media_url IS NOT NULL
AND download_attempts < 3
LIMIT %d",
@@ -440,8 +584,14 @@ class MLS_Media_Handler {
foreach ($pending as $media) {
if ($this->download_media($media->id)) {
$stats['success']++;
if ($progress_callback) {
call_user_func($progress_callback, 'media_downloaded', array('media_key' => $media->media_key));
}
} else {
$stats['failed']++;
if ($progress_callback) {
call_user_func($progress_callback, 'media_error', array('media_key' => $media->media_key));
}
}
}
wp-content/plugins/mls-by-hansonxyz/includes/class-mls-sync-engine.php
+131 -13
View File
@@ -88,6 +88,9 @@ class MLS_Sync_Engine {
public function run_full_sync($dry_run = false, $limit = null, $progress_callback = null) {
$this->logger->info('Starting full sync', array('dry_run' => $dry_run, 'limit' => $limit));
// Store progress callback for use in process_property
$this->progress_callback = $progress_callback;
// Create sync state record
if (!$dry_run) {
$this->sync_state_id = $this->create_sync_state(self::TYPE_FULL);
@@ -104,12 +107,35 @@ class MLS_Sync_Engine {
try {
// Get first page of properties with media
$start_time = microtime(true);
$this->emit_progress('api_request', array(
'method' => 'GET',
'url' => 'Property',
'params' => array('type' => 'full_sync', 'limit' => $limit),
));
$response = $this->api_client->get_properties_for_sync(null, 'Media', $limit ? min($limit, 1000) : null);
$elapsed = round((microtime(true) - $start_time) * 1000);
if (is_wp_error($response)) {
$this->emit_progress('api_response', array(
'success' => false,
'status_code' => 0,
'error' => $response->get_error_message(),
'record_count' => 0,
'response_time' => $elapsed,
));
throw new Exception($response->get_error_message());
}
$this->emit_progress('api_response', array(
'success' => true,
'status_code' => 200,
'record_count' => isset($response['value']) ? count($response['value']) : 0,
'response_time' => $elapsed,
'has_more' => isset($response['@odata.nextLink']),
));
// Process pages
$continue = true;
while ($continue && isset($response['value'])) {
@@ -120,12 +146,11 @@ class MLS_Sync_Engine {
}
$this->process_property($property, $dry_run);
if ($progress_callback) {
call_user_func($progress_callback, $this->stats);
}
}
// Emit page complete event
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
// Check for next page
if ($continue && isset($response['@odata.nextLink'])) {
// Save progress
@@ -138,11 +163,34 @@ class MLS_Sync_Engine {
));
}
$start_time = microtime(true);
$this->emit_progress('api_request', array(
'method' => 'GET',
'url' => 'Property (next page)',
'params' => array('page' => 'next'),
));
$response = $this->api_client->get_next_page($response['@odata.nextLink']);
$elapsed = round((microtime(true) - $start_time) * 1000);
if (is_wp_error($response)) {
$this->emit_progress('api_response', array(
'success' => false,
'status_code' => 0,
'error' => $response->get_error_message(),
'record_count' => 0,
'response_time' => $elapsed,
));
throw new Exception($response->get_error_message());
}
$this->emit_progress('api_response', array(
'success' => true,
'status_code' => 200,
'record_count' => isset($response['value']) ? count($response['value']) : 0,
'response_time' => $elapsed,
'has_more' => isset($response['@odata.nextLink']),
));
} else {
$continue = false;
}
@@ -211,6 +259,9 @@ class MLS_Sync_Engine {
'dry_run' => $dry_run,
));
// Store progress callback for use in process_property
$this->progress_callback = $progress_callback;
if (!$dry_run) {
$this->sync_state_id = $this->create_sync_state(self::TYPE_INCREMENTAL);
$this->logger->set_sync_state($this->sync_state_id);
@@ -226,29 +277,74 @@ class MLS_Sync_Engine {
try {
// Get modified properties (including those marked for deletion)
$start_time = microtime(true);
$this->emit_progress('api_request', array(
'method' => 'GET',
'url' => 'Property',
'params' => array('type' => 'incremental', 'since' => $last_timestamp),
));
$response = $this->api_client->get_properties_since($last_timestamp, 'Media');
$elapsed = round((microtime(true) - $start_time) * 1000);
if (is_wp_error($response)) {
$this->emit_progress('api_response', array(
'success' => false,
'status_code' => 0,
'error' => $response->get_error_message(),
'record_count' => 0,
'response_time' => $elapsed,
));
throw new Exception($response->get_error_message());
}
$this->emit_progress('api_response', array(
'success' => true,
'status_code' => 200,
'record_count' => isset($response['value']) ? count($response['value']) : 0,
'response_time' => $elapsed,
'has_more' => isset($response['@odata.nextLink']),
));
// Process pages
while (isset($response['value'])) {
foreach ($response['value'] as $property) {
$this->process_property($property, $dry_run);
if ($progress_callback) {
call_user_func($progress_callback, $this->stats);
}
}
// Emit page complete event
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
// Check for next page
if (isset($response['@odata.nextLink'])) {
$start_time = microtime(true);
$this->emit_progress('api_request', array(
'method' => 'GET',
'url' => 'Property (next page)',
'params' => array('page' => 'next'),
));
$response = $this->api_client->get_next_page($response['@odata.nextLink']);
$elapsed = round((microtime(true) - $start_time) * 1000);
if (is_wp_error($response)) {
$this->emit_progress('api_response', array(
'success' => false,
'status_code' => 0,
'error' => $response->get_error_message(),
'record_count' => 0,
'response_time' => $elapsed,
));
throw new Exception($response->get_error_message());
}
$this->emit_progress('api_response', array(
'success' => true,
'status_code' => 200,
'record_count' => isset($response['value']) ? count($response['value']) : 0,
'response_time' => $elapsed,
'has_more' => isset($response['@odata.nextLink']),
));
} else {
break;
}
@@ -324,6 +420,7 @@ class MLS_Sync_Engine {
}
$this->sync_state_id = $sync_state_id;
$this->progress_callback = $progress_callback;
$this->logger->set_sync_state($sync_state_id);
$this->logger->info('Resuming sync', array('sync_state_id' => $sync_state_id));
@@ -359,12 +456,11 @@ class MLS_Sync_Engine {
while (isset($response['value'])) {
foreach ($response['value'] as $property) {
$this->process_property($property, false);
if ($progress_callback) {
call_user_func($progress_callback, $this->stats);
}
}
// Emit page complete event
$this->emit_progress('page_complete', array('processed' => $this->stats['processed']));
if (isset($response['@odata.nextLink'])) {
$this->update_sync_state(array(
'last_next_link' => $response['@odata.nextLink'],
@@ -413,6 +509,11 @@ class MLS_Sync_Engine {
);
}
/**
* Progress callback reference
*/
private $progress_callback = null;
/**
* Process a single property record
*
@@ -439,6 +540,7 @@ class MLS_Sync_Engine {
$this->delete_property($listing_key);
}
$this->stats['deleted']++;
$this->emit_progress('property_deleted', array('listing_key' => $listing_key));
return;
}
@@ -454,8 +556,10 @@ class MLS_Sync_Engine {
if ($dry_run) {
if ($existing) {
$this->stats['updated']++;
$this->emit_progress('property_skipped', array('listing_key' => $listing_key));
} else {
$this->stats['created']++;
$this->emit_progress('property_skipped', array('listing_key' => $listing_key));
}
return;
}
@@ -468,17 +572,31 @@ class MLS_Sync_Engine {
array('listing_key' => $listing_key)
);
$this->stats['updated']++;
$this->emit_progress('property_updated', array('listing_key' => $listing_key));
} else {
// Insert new
$data['listing_key'] = $listing_key;
$data['created_at'] = current_time('mysql');
$wpdb->insert($this->db->properties_table(), $data);
$this->stats['created']++;
$this->emit_progress('property_created', array('listing_key' => $listing_key));
}
// Process media if present
if (isset($property['Media']) && is_array($property['Media'])) {
$this->media_handler->sync_property_media($listing_key, $property['Media']);
$this->media_handler->sync_property_media($listing_key, $property['Media'], false, $this->progress_callback);
}
}
/**
* Emit progress event
*
* @param string $event Event name
* @param array $data Event data
*/
private function emit_progress($event, $data = array()) {
if ($this->progress_callback) {
call_user_func($this->progress_callback, $event, $data);
}
}